R Notebook

0.1 Introduction

For our second guided analysis, we will be working on advancing the search for a model to identify aggregate test score anomalies in our state. In our previous module we determined that a one-size-fits-all approach led to a model that mostly identified schools in lower grades, and most often identified schools with small class sizes. Even with the addition of extra data, the data team has not yet found a model that identifies abnormalities that are not correlated with school size and school grade level.

You have been given the time to explore some alternative approaches to this work - including stepping away from the one-size-fits-all approach and moving toward a multi-model approach. Your supervisor has told you that before considering the task not possible with the data at hand, she’d like you to look at the possibility of fitting separate models to separate samples of the data and seeing what this changes.

0.1.1 Setup and Read Data

For this task, the data team now has a full dataset available to it - IT has filled each of the requests for additional data. Our first task is to join these datasets and create an analysis dataset.

library(dplyr)
# Read in the data
# stu_data <- read.csv("data/sim_assessment_data.csv", stringsAsFactors = FALSE)
load("data/cohort_test_scores.rda")
head(sch_score, addrownums = FALSE)

load("data/agency.rda")
load("data/attend.rda")
load("data/expel.rda")
load("data/enrollment.rda")
load("data/staff.rda")
ls()

 [1] "adj_model"         "agency"            "alt_model"         "attend"            "b1"               
 [6] "boot_dat"          "by_group"          "cluster_vcov"      "coefplot_coeftest" "custom_brks"      
[11] "cv_dat"            "dffits_limit"      "example_districts" "example_school"    "expel"            
[16] "ftests"            "full_data"         "full_enroll"       "get_CL_vcov"       "glance"           
[21] "m_ex"              "m_notrobust"       "m_robust"          "m1"                "m1_inf"           
[26] "m2"                "m3"                "m4"                "m5"                "mod_test_distid"  
[31] "mod_test_ss1"      "models1"           "models2"           "models3"           "N"                
[36] "p1"                "p2"                "plotdf"            "predict.robust"    "regress_battery"  
[41] "sch_score"         "sch_score_m1"      "simple_anova"      "simple_model"      "staff"            
[46] "tmp_fit"           "unweighted_mod"    "vam_model"         "weighted_mod"      "x1"               
[51] "y"

full_data <- inner_join(sch_score, agency, 
                        by = c("distid", "schid", "test_year"))
full_data <- inner_join(full_data, attend, 
                        by = c("distid", "schid", "test_year", "school_year"))
full_data <- inner_join(full_data, expel, 
                        by = c("distid", "schid", "test_year", "school_year"))
full_data <- inner_join(full_data %>% dplyr::select(-enrollment), full_enroll, 
                        by = c("distid", "schid", "test_year", "school_year"))
full_data <- inner_join(full_data, staff, 
                        by = c("distid", "schid", "test_year", "school_year"))
str(full_data)

'data.frame':   15087 obs. of  69 variables:
 $ distid                    : num  275 222 288 287 174 290 148 88 174 174 ...
 $ schid                     : int  9 1 11 10 120 3 2 1 129 64 ...
 $ test_year                 : num  2007 2010 2007 2011 2011 ...
 $ grade                     : num  7 4 4 4 4 7 5 8 4 5 ...
 $ subject                   : chr  "read" "read" "math" "math" ...
 $ n1                        : num  195 107 39 12 42 89 72 111 10 20 ...
 $ ss1                       : num  1213 1115 966 999 931 ...
 $ n2                        : num  199 109 41 12 33 75 76 108 21 36 ...
 $ ss2                       : num  1233 1166 1063 1071 1015 ...
 $ school_year               : chr  "2006-07" "2009-10" "2006-07" "2010-11" ...
 $ cesa                      : int  1 2 9 4 1 3 10 1 1 1 ...
 $ county                    : int  40 13 37 32 40 25 9 40 40 40 ...
 $ athl_conf                 : int  47 158 46 28 27 42 4 47 27 27 ...
 $ locale                    : int  3 4 2 2 1 4 4 3 1 1 ...
 $ locale_desc               : chr  "Urban Fringe of a Large City" "Urban Fringe of a Mid-Size City" "Mid-Size City" "Mid-Size City" ...
 $ agency_type               : chr  "Public school" "Public school" "Public school" "Public school" ...
 $ grade_group               : chr  "Middle School" "Elementary School" "Elementary School" "Elementary School" ...
 $ low_grade                 : chr  "06" "03" "KG" "K4" ...
 $ high_grade                : chr  "08" "05" "05" "08" ...
 $ school_size               : chr  "Large" "Medium" "Medium" "Small" ...
 $ charter_ind               : chr  "No" "No" "No" "Yes" ...
 $ title1a_program           : chr  "TASNF" "TAS" "TAS" "SWP" ...
 $ title1a_program_desc      : chr  "Targeted Assistance eligible but not funded" "Targeted Assistance eligible and funded" "Targeted Assistance eligible and funded" "School-wide Program" ...
 $ days_possible             : num  107260 76496 53705 21606 72782 ...
 $ days_attended             : num  103245 73609 51537 20568 67523 ...
 $ att_rate                  : num  96.3 96.2 96 95.2 92.8 ...
 $ total_enroll              : int  614 445 310 133 418 268 505 287 349 353 ...
 $ suspension_rate           : num  10.9 NA 2.9 NA 14.3 ...
 $ suspension_count          : num  67 0 9 0 60 0 2 4 204 102 ...
 $ susp_rate_female          : num  4.12 NA 0.6 NA 10.81 ...
 $ susp_rate_male            : num  17.03 NA 5.59 NA 17.17 ...
 $ susp_count_female         : num  12 0 1 0 20 0 1 1 70 25 ...
 $ susp_count_male           : num  55 0 8 0 40 0 1 3 134 77 ...
 $ amer_ind_enroll           : int  3 2 1 1 1 1 6 5 2 2 ...
 $ asian_enroll              : int  42 13 52 9 132 2 7 18 2 16 ...
 $ black_enroll              : int  104 13 15 0 232 5 14 59 336 284 ...
 $ hispanic_enroll           : int  28 16 9 5 24 4 8 9 0 11 ...
 $ mutli_enroll              : int  0 0 0 11 0 0 0 0 0 0 ...
 $ pac_island_enroll         : int  0 0 0 0 0 0 0 0 0 0 ...
 $ white_enroll              : int  437 401 233 107 29 256 470 196 9 40 ...
 $ amer_ind_susp             : num  NA NA NA NA NA NA 0 NA NA NA ...
 $ asian_susp                : num  1 NA 1 0 1 NA 0 0 NA 0 ...
 $ black_susp                : num  25 NA 3 0 55 NA 0 4 198 92 ...
 $ hispanic_susp             : num  NA 0 NA NA NA NA 0 NA 0 NA ...
 $ multi_susp                : num  0 0 0 0 0 0 0 0 0 0 ...
 $ pac_island_susp           : num  0 0 0 0 0 0 0 0 0 0 ...
 $ white_susp                : num  35 0 4 0 3 0 2 0 NA 7 ...
 $ enrollment                : num  612 445 311 133 418 268 505 287 349 353 ...
 $ frl_per                   : num  17.6 16.6 45.7 36.1 82.1 ...
 $ non_frl_per               : num  82.3 83.4 54.3 63.9 17.9 ...
 $ frl_count                 : num  108 74 142 48 343 71 282 29 328 296 ...
 $ non_frl_count             : num  504 371 169 85 75 197 223 258 21 57 ...
 $ swd_per                   : num  9.64 14.38 7.72 4.51 14.11 ...
 $ non_swd_per               : num  90.4 85.6 92.3 95.5 85.9 ...
 $ swd_count                 : num  59 64 24 6 59 49 79 34 57 69 ...
 $ non_swd_count             : num  553 381 287 127 359 219 426 253 292 284 ...
 $ ell_per                   : num  5.23 2.7 19.94 NA 20.81 ...
 $ non_ell_per               : num  94.8 97.3 80.1 100 79.2 ...
 $ ell_count                 : num  32 12 62 NA 87 2 2 26 NA 2 ...
 $ non_ell_count             : num  580 433 249 133 331 266 503 261 349 351 ...
 $ total_enrollment          : int  614 445 310 133 418 268 505 287 349 353 ...
 $ admin_fte                 : num  2 1 0.8 0.33 1 1 1 0.7 2 1.5 ...
 $ ratio_stu_to_admin        : num  307 445 388 403 418 ...
 $ support_staff_fte         : num  14.51 13.93 15.78 0.79 9.95 ...
 $ ratio_stu_to_supstaff     : num  42.3 31.9 19.6 168.3 42 ...
 $ licensed_fte              : num  47.9 38.71 28.67 9.43 30.48 ...
 $ ratio_stu_to_licensed     : num  12.8 11.5 10.8 14.1 13.7 ...
 $ total_staff               : num  64.4 53.6 45.2 10.6 41.4 ...
 $ ratio_students_to_allstaff: num  9.53 8.3 6.85 12.61 10.09 ...

rm(agency, attend, sch_score, expel, staff, full_enroll)

Despite the additional data - there is pressure to keep the model simple. Your team feels it will be easier to explain to external stakeholders and will create fewer opportunities for schools to dispute their claims. The colleague who proposed the first one-size-fits-all has proposed a simple alternative in response to your feedback and subsequent analyses.

The new model is to simply fit a separate regression of the lagged scale score on the current scale score for each separate grade-subject-year combination. Your colleague has given you the code below to iterate through your data and fit these combinations.

First, let’s see how many models this gives us:

full_data %>% dplyr::select(test_year, grade, subject) %>% 
  distinct %>% nrow

[1] 50

And, how many rows per model:

full_data %>% dplyr::select(test_year, grade, subject) %>% 
  group_by(test_year, grade, subject) %>%
  summarize(count = n()) %>% 
  pull(count) %>% summary

   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
  152.0   220.5   253.0   301.7   425.8   469.0

We have as few as 152 observations in some groups and as many as 469. Since we know regression models are efficient and do not have large data requirements, this is not a concern.

Now, let’s look at a loop we can use to fit the same model to each of our 50 subsets of data efficiently:

# Load the packages we need to fit multiple models
library(tidyverse)

[37m-- [1mAttaching packages[22m --------------------------------------- tidyverse 1.2.1 --[39m
[37m[32mv[37m [34mggplot2[37m 3.0.0     [32mv[37m [34mreadr  [37m 1.1.1
[32mv[37m [34mtibble [37m 1.4.2     [32mv[37m [34mpurrr  [37m 0.2.5
[32mv[37m [34mtidyr  [37m 0.8.1     [32mv[37m [34mstringr[37m 1.3.1
[32mv[37m [34mggplot2[37m 3.0.0     [32mv[37m [34mforcats[37m 0.3.0[39m
[37m-- [1mConflicts[22m ------------------------------------------ tidyverse_conflicts() --
[31mx[37m [34mdplyr[37m::[32mfilter()[37m masks [34mstats[37m::filter()
[31mx[37m [34mdplyr[37m::[32mlag()[37m    masks [34mstats[37m::lag()[39m

library(broom) # to manipulate regression models
library(modelr) # to fit multiple models and get their attributes easily


Attaching package: 㤼㸱modelr㤼㸲

The following object is masked from 㤼㸱package:broom㤼㸲:

    bootstrap

# Define a grouped dataframe using only the columns we need
# group by the grade, subject, and year
# Then nest so the data hangs together in one list
by_group <- full_data[, 1:10] %>%
  group_by(grade, subject, test_year) %>%
  nest()

# Define a simple function that fits our model
simple_model <- function(df) {
  lm(ss2 ~ ss1, data = df)
}

# Apply that model to each group in our dataset using the `map` function
# This is equivalent to a loop, but more efficient to write
by_group <- by_group %>%
  mutate(model = map(data, simple_model))

# To understand our model(s) - use the `glance` function to compute some 
# summary statistics for each model
glance <- by_group %>%
  mutate(glance = map(model, broom::glance)) %>%
  unnest(glance, .drop = TRUE)

# Display the output
arrange(glance, desc(r.squared))

There is a lot to unpack in this bit of code. We’re going to explore it in detail and then modify different pieces of it to do our analysis. This is where the iterative power of statistical computing really shines, because we can do many operations repeatedly across our sub-samples, collect the results, and compare and compute the results of our analyses.

The first thing we want to do is gather our data into subsamples grouped by the grade, subject, and year of the test. This code takes our full data set, groups it by these variables, and then nests each observation for each group into a new dataframe and stores that dataframe as a column. The resulting data structure is a dataframe with 50 rows, but one of the columns contains a dataset within it that corresponds to the full observations for that combination of grade, subject, and test year.

It’s a neat trick that is useful in education data analysis where so many of our questions are about nested structures of observations - students in classrooms, classrooms in schools, schools in districts, etc.

by_group <- full_data[, 1:10] %>%
  group_by(grade, subject, test_year) %>%
  nest()

head(by_group)

The advantage here is that we already know how to tell R to do things to an entire column of data at once, so we can use those same skills to rapidly make calculations on each of these fifty separate datasets all without having to write an explicit loop.

The next set of code defines our model as a function called, uncreatively, simple_model. Function definitions are easy in R, once you read in the function definition, you will see it appear in the “Functions” section of your RStudio environment pane, and you can use it just like any other command in R.

Writing functions can be a bit tricky at first. This function is not ideal because it has variables within it (ss2 and ss1) that are not defined in the function itself – it assumes they will exist. It is safe for us to use here because we know those variables will always exist.

The next line of code simply tells R to create a new column in our dataset called model, and to create that dataset it should apply the function simple_model to each element of the data column and store the output (a linear model) in the new column. The map function is from the purrr package and tells R to use the simple_model function on each element of the data column. The term map refers to the fact that we are “mapping” the function simple_model to each element of the data column.

simple_model <- function(df) {
  # df is a user specified parameter to the function
  # ss2 and ss1 are variables that must be present in the df provided by the user
  lm(ss2 ~ ss1, data = df)
}

head(by_group)


by_group <- by_group %>%
  # take each element of the data column, and pass it individually as the 
  # first argument to the simple_model function
  mutate(model = map(data, simple_model))

head(by_group)

NA

So now we have created for each group two new columns, but instead of storing values, they store a dataset and a statistical model respectively. Go ahead and look for yourself:

summary(by_group$data[[10]]) # 10 and 11 are just indexes that say which e

     distid          schid              n1              ss1               n2              ss2        
 Min.   :  2.0   Min.   :  1.00   Min.   :  6.00   Min.   : 910.1   Min.   :  3.00   Min.   : 950.8  
 1st Qu.: 87.0   1st Qu.:  2.00   1st Qu.: 35.00   1st Qu.:1075.5   1st Qu.: 36.00   1st Qu.:1124.0  
 Median :174.0   Median :  5.00   Median : 52.00   Median :1101.5   Median : 52.00   Median :1158.7  
 Mean   :173.6   Mean   : 18.32   Mean   : 54.42   Mean   :1098.0   Mean   : 55.97   Mean   :1155.5  
 3rd Qu.:254.0   3rd Qu.: 17.00   3rd Qu.: 69.00   3rd Qu.:1130.9   3rd Qu.: 70.00   3rd Qu.:1189.5  
 Max.   :355.0   Max.   :137.00   Max.   :211.00   Max.   :1215.5   Max.   :239.00   Max.   :1279.5  
 school_year       
 Length:425        
 Class :character  
 Mode  :character

#                              element we want, here the 10th element
summary(by_group$data[[11]]) # here the 11th element

     distid          schid              n1             ss1             n2             ss2       school_year       
 Min.   :  1.0   Min.   :  1.00   Min.   : 10.0   Min.   :1089   Min.   :  5.0   Min.   :1026   Length:253        
 1st Qu.:105.0   1st Qu.:  1.00   1st Qu.: 48.0   1st Qu.:1219   1st Qu.: 52.0   1st Qu.:1248   Class :character  
 Median :174.0   Median :  3.00   Median :112.0   Median :1252   Median :111.0   Median :1277   Mode  :character  
 Mean   :179.2   Mean   : 19.45   Mean   :122.2   Mean   :1241   Mean   :124.8   Mean   :1267                     
 3rd Qu.:248.0   3rd Qu.: 15.00   3rd Qu.:182.0   3rd Qu.:1278   3rd Qu.:185.0   3rd Qu.:1303                     
 Max.   :356.0   Max.   :136.00   Max.   :379.0   Max.   :1356   Max.   :393.0   Max.   :1388

summary(by_group$model[[1]]) # here the first element


Call:
lm(formula = ss2 ~ ss1, data = df)

Residuals:
     Min       1Q   Median       3Q      Max 
-113.135   -7.867    2.451    9.649   34.365 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept) 118.6035    24.1043    4.92 1.59e-06 ***
ss1           0.9232     0.0207   44.60  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 17.18 on 243 degrees of freedom
Multiple R-squared:  0.8911,    Adjusted R-squared:  0.8907 
F-statistic:  1989 on 1 and 243 DF,  p-value: < 2.2e-16

summary(by_group$model[[2]])


Call:
lm(formula = ss2 ~ ss1, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-80.218 -12.039   0.017  12.578  81.371 

Coefficients:
             Estimate Std. Error t value Pr(>|t|)    
(Intercept) -83.50504   26.08640  -3.201  0.00148 ** 
ss1           1.11807    0.02453  45.573  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 20.59 on 408 degrees of freedom
Multiple R-squared:  0.8358,    Adjusted R-squared:  0.8354 
F-statistic:  2077 on 1 and 408 DF,  p-value: < 2.2e-16

Pretty neat! But, it’s not enough to calculate 50 regression models, we want to test, compare, and perhaps modify them. With the way our data are now structured, we can easily extract the features we are interested in from each of the models and store them for further analysis. Let’s say we wanted to look at how the intercept and slope terms vary across the fifty groups, we can do that!

Again we use the map function to extract the coefficients from our model - in this case, we will have two coefficients. We use the R function coef and we map it to each element of model.


by_group <- inner_join(
  by_group, # our original data
   by_group %>%
      mutate(coefs = map(model, coef)) %>%  # get the coefficients out of the model
      group_by(grade, subject, test_year) %>% # group the data 
      do(data.frame(t(unlist(.$coefs)))) # split each coefficient into a new column
)

Joining, by = c("grade", "subject", "test_year")

names(by_group)

[1] "grade"        "subject"      "test_year"    "data"         "model"        "X.Intercept." "ss1"

# Replace the 6th elements name
names(by_group)[6] <- "est_intercept"
# Replace the 7th elements name
names(by_group)[7] <- "est_ss1"

And now we can plot the differences and do some graphical EDA.

ggplot(by_group, aes(x = est_intercept)) + geom_density() + theme_bw()

ggplot(by_group, aes(x = est_ss1)) + geom_density() + theme_bw()


ggplot(by_group) + scale_x_continuous(limits = c(700, 1600)) +
  scale_y_continuous(limits = c(700, 1600)) + 
  geom_abline(data = by_group, 
              aes(slope = est_ss1, intercept = est_intercept), 
              alpha = I(0.5)) + theme_bw()

OK, so we’ve got our basic strategy. First, we’re going to explore some methods for model comparison. Then we’re going to look at some empirical tests we can apply to models. Then we’ll look at methods for demonstrating and communicating models to the wider world.

0.2 Residuals

It can be helpful to visually inspect the residuals, as we have seen previously, not just to diagnose misfit, but to get a sense of the accuracy of our model. We can create residual plots combining the fits from all of the models relatively easily with another set of looping code.

by_group <- by_group %>%
  mutate(
    resids = map2(data, model, add_residuals),
    preds = map2(data, model, add_predictions)
  )
by_group


# unnest expands the data out from a column
plotdf <- left_join(unnest(by_group, resids),
                    unnest(by_group, preds))

Joining, by = c("grade", "subject", "test_year", "est_intercept", "est_ss1", "distid", "schid", "n1", "ss1", "n2", "ss2", "school_year")

# Residual plot

plotdf %>%
  ggplot(aes(x = pred, y = resid)) +
  geom_point(alpha = I(0.2)) + theme_bw() +
  geom_smooth()


# Residual segment plot
plotdf %>% top_n(200) %>% 
  ggplot(aes(x = ss1, xend = ss1, y = pred, yend = ss1)) +
  geom_segment(alpha = I(0.2)) + theme_bw() +
  geom_smooth()

Selecting by pred

# Residual arrow plot
plotdf %>% sample_n(50) %>% 
  ggplot(aes(x = ss1, xend = ss1, y = ss2, yend = pred)) + 
    geom_segment(arrow = arrow(angle = 15, length = unit(0.1, "inches")), lineend = "butt")  + 
    geom_smooth(aes(x = ss1, y = ss2), se=FALSE) + 
      cowplot::theme_cowplot(font_size = 16) + 
  geom_point(aes(x = ss1, y = ss2)) + 
  labs(title = "Residuals Plotted by Observed Values of Pre-Test", 
       caption = "Arrows show predicted value compared to observed value point.", 
       x = "Pre-Test", y = "Posttest", 
       subtitle = "Residuals from 50 year-subject-grade models of pretest on posttest")

We notice that even when we have fit 50 models to different sets of the data our models exhibit the funneling as the predicted value increases. We already know this is a symptom of model misspecification.

Further inspection of the residuals is helpful here - so let’s look at some formal ways of identifying outliers, high leverage, and influential observations next.

0.3 Outliers, High Leverage, and Influential Observations

As we have mentioned, OLS regression models can be sensitive to outliers. There are a number of diagnostic techniques that can be used to identify outliers.

All quotes below and excellent further detail on regression diagnostics is available through the free, online, Penn State course on regression - available here: https://onlinecourses.science.psu.edu/stat501/node/338

A common method to measure potential outliers is to compute the “hat values” or “leverage” for each observation in a regression model.

Leverages are defined as:

the leverage, $h_{ii}$, quantifies the influence that the observed response $y_{i}$ has on its predicted value $\hat y_{i}$. That is, if $h_{ii}$ is small, then the observed response $y_{i}$ plays only a small role in the value of the predicted response $\hat y_{i}$. On the other hand, if $h_{ii}$ is large, then the observed response $y_{i} plays a large role in the value of the predicted response $\hat y_{i}$. It’s for this reason that the $h_{ii}$ are called the “leverages.”

Leverages can be interpreted as the distance between the x values of an observation and the mean of the x values for all remaining data points. Leverages have range from 0 to 1 and the sum of all of the leverages is equal to the total number of parameters (regression coefficients, including the intercept) in the model.

A word of caution! Remember, a data point has large influence only if it affects the estimated regression function. Leverages only take into account the extremeness of the x values, but a high leverage observation may or may not actually be influential.

There is such an important distinction between a data point that has high leverage and one that has high influence that it is worth saying it one more time:

The leverage merely quantifies the potential for a data point to exert strong influence on the regression analysis. The leverage depends only on the predictor values. Whether the data point is influential or not also depends on the observed value of the reponse $y_{i}$.

0.3.1 High Leverage

To better understand our data, let’s start by looking at observations with a high potential to influence our model – high-leverage observations. To compute the leverages, we can use the augment() function in the broom package, which adds observation-level regression diagnostics to our original data frame for convenient analysis and plotting.

For simplicity, we’ll fit one global model and work with it. At the end, we’ll return to how to apply this analysis to submodels.

library(broom) # R package for working with regression models smoothly
# Create a new dataset with augmented values
m1 <- lm(ss2 ~ ss1 + n1 + subject, data = full_data)
sch_score_m1 <- augment(m1) # generate the leverages and more for each row in data
names(sch_score_m1)

 [1] "ss2"        "ss1"        "n1"         "subject"    ".fitted"    ".se.fit"    ".resid"     ".hat"      
 [9] ".sigma"     ".cooksd"    ".std.resid"

We have our original outcome variable, predictor variable, our model fits, the standard error of our model fits, the residuals, and then a series of diagnostic variables: .hat, .sigma, .cooksd, and .std.resid. For now, we will focus on the .hat variable as this is the measure of leverage for each observation. One conventional rule of thumb is that an observation with a hat value greater than 3 times the mean hat value is potentially influential. Let’s compute a table of how many observations meet this criteria.

table(sch_score_m1$.hat > mean(sch_score_m1$.hat)*3)


FALSE  TRUE 
14725   362

We have a big dataset, so while 362 is a large number of observations, proportionally it is fairly small. This is a typical result for a dataset this size and a model of this nature.

Bonus question: How can you verify that this amount is typical?

Remember that our research question is around the outliers though - we want the model to identify meaningful outliers, so paying careful attention to the attributes of these observations is important.

We can get a little more detail by calculating the ratio of each hat value to the mean of all the hat values. The code below rounds this ratio to the nearest tenth and presents it in a table, allowing us to get a fuller picture of the whole distribution of hat value - remember that values greater than 3x the mean are considered high leverage.

# Expanded code to make it easier to read:
knitr::kable( # to make a pretty table in our document
  table( # compute a table
    round( # round the result, here to the tenths (1)
      sch_score_m1$.hat/mean(sch_score_m1$.hat), 1
      )
    )
  )

Var1	Freq
0.5	1601
0.6	3275
0.7	2428
0.8	1680
0.9	1172
1	808
1.1	696
1.2	543
1.3	375
1.4	337
1.5	340
1.6	245
1.7	183
1.8	177
1.9	168
2	95
2.1	110
2.2	91
2.3	82
2.4	62
2.5	73
2.6	52
2.7	54
2.8	43
2.9	24
3	28
3.1	38
3.2	14
3.3	22
3.4	21
3.5	20
3.6	13
3.7	19
3.8	12
3.9	14
4	16
4.1	14
4.2	10
4.3	15
4.4	10
4.5	11
4.6	4
4.7	10
4.8	4
4.9	5
5	2
5.1	6
5.2	4
5.3	4
5.4	2
5.5	4
5.6	6
5.7	6
5.8	1
5.9	2
6	4
6.1	4
6.2	1
6.3	3
6.4	1
6.5	1
6.7	1
6.8	2
7	5
7.1	2
7.2	1
7.4	1
7.5	1
7.6	1
7.7	1
7.8	1
8	1
8.2	1
8.6	1
8.8	1
8.9	1
9	1

Now that we have a sense of the distribution of high leverage observations and how many we have (how would you describe this distribution?) - it’s time to look at the impact they have on the regression model itself by directly measuring their influence.

0.3.1.1 Influential Observations

Once we have identified our high leverage observations, we can combine this information with information about the influence of observations - incorporating information about the dependent variable as well. A quick first look at this is to look at the hat values that also have large (or small) values of the outcome variable. These observations, mathematically, will exert extra influence on our estimate of beta, and are thus influential on our model.

This code simulates a perfectly specified regression model.

# Generate an ideal high leverage plot
set.seed(52523)
# Set values to simulate the data
N <- 15000
x1 <- rnorm(N)
b1 <- rnorm(N, 0.8, 0.25)
# a1 <- rbinom(N, size = 1, prob = 0.7)
y <- 6 + b1 * x1 + rnorm(N, 0, 0.5)
m_ex <- lm(y ~ x1)
plotdf <- augment(m_ex)
ggplot(plotdf, aes(x = y, y = .hat)) + geom_point(alpha = 1/5) + theme_bw() + 
  geom_hline(yintercept = 3 * mean(plotdf$.hat))

In a well-specified regression model the high leverage observations will be evenly balanced between high and low values of the outcome, as in the plot above. Think about the points above the high-leverage horizontal line as balancing against one another – too many to one side or another and the results of the model may be skewed and at risk of being biased by outlier observations.

Let’s see how the model we fit looks by this metric:

ggplot(sch_score_m1, aes(x = ss2, y = .hat)) + 
  geom_point(alpha = 1/5) + 
  theme_bw() + 
  geom_smooth(se=FALSE) + 
  geom_hline(yintercept = 3 * mean(sch_score_m1$.hat)) # generate a reference line

We don’t have to rely on the eye-test alone. Fortunately, we have a further set of diagnostics available to use to measure the influence that each observation has on our estimates of $\beta$. Collectively known as influence measures, these compute the influence of individual observations by looking at the difference in the model without each observation, compared to the model with all observations. This deletion technique allows us to empirically assess the contribution of each observation to the coefficients in the model. This metric is called a DFBETA. The dfbeta for each observation is calculated for each regressor in our dataset, so with three regressors and an intercept we will get 4 DFBETA measures.

R provides us a convenience function that computes the deletion measures for us. The influence.measures() returns a list of matrices computing these features.

m1_inf <- influence.measures(m1)
head(m1_inf$infmat)

         dfb.1_       dfb.ss1        dfb.n1      dfb.sbjc         dffit    cov.r       cook.d          hat
1  4.264909e-04 -1.871738e-04 -2.390392e-03 -1.545492e-03 -3.651777e-03 1.000631 3.334081e-06 0.0003749864
2  1.943648e-03 -2.193774e-03  3.561255e-03  5.878008e-03  8.855933e-03 1.000294 1.960755e-05 0.0001592070
3  1.101644e-02 -1.000475e-02  2.187796e-03 -5.345136e-03  1.356193e-02 1.000466 4.598290e-05 0.0003430263
4  3.832584e-05 -3.137221e-05 -1.262509e-05 -3.106272e-05  6.571982e-05 1.000529 1.079845e-09 0.0002636500
5 -1.337187e-05  1.245743e-05 -3.866228e-06  4.893462e-06 -1.512928e-05 1.000740 5.722755e-11 0.0004739941
6 -6.156281e-03  6.109131e-03 -2.230218e-03  5.248715e-03  1.007478e-02 1.000355 2.537620e-05 0.0002150981

This format isn’t particularly helpful to us - it is hard to interpret these numbers. Let’s focus on the coefficient for ss1 given by the column dfb.ss1. This represents the change in the coefficient with this observation excluded, but the change is scaled by the standard error of the coefficient itself. Let’s do a quick worked example to understand how the scale of this important measure works.

First, let’s remind ourselves of the standard error and estimate of our coefficient for ss1.

summary(m1)


Call:
lm(formula = ss2 ~ ss1 + n1 + subject, data = full_data)

Residuals:
     Min       1Q   Median       3Q      Max 
-184.629  -14.524    1.241   15.584  140.913 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) 218.408614   2.992677   72.98   <2e-16 ***
ss1           0.852893   0.002806  303.98   <2e-16 ***
n1            0.054507   0.003768   14.46   <2e-16 ***
subjectread -26.382956   0.400526  -65.87   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 24.55 on 15083 degrees of freedom
Multiple R-squared:  0.9007,    Adjusted R-squared:  0.9007 
F-statistic: 4.561e+04 on 3 and 15083 DF,  p-value: < 2.2e-16

The coefficient is 0.852893 and the standard error is ~0.003. Now, let’s look at the largest and smallest values of DFBETAS:

max(m1_inf$infmat[, 2] )

[1] 0.1056855

min(m1_inf$infmat[, 2] )

[1] -0.1154258

To see the change in the slope on ss1 that results, we need to do some quick math – first we “unstandardize” the estimate by multiplying by the standard error:

max(m1_inf$infmat[, 2] ) * 0.003

[1] 0.0003170564

min(m1_inf$infmat[, 2] ) * 0.003

[1] -0.0003462773

And then we add this to the coefficient to see how much the coefficient will change:

(max(m1_inf$infmat[, 2] ) * 0.003) + 0.852893

[1] 0.8532101

(min(m1_inf$infmat[, 2] ) * 0.003) + 0.852893

[1] 0.8525467

The change is not very much for our biggest and smallest values, but remember that this is because there are > 15,000 observations in our data, so we need to focus on the impact of observations relative to the average observation, not their impact alone. A common rule of thumb for identifying large DFBETA values is any value larger than 2 / sqrt(n) where n is the number of rows in our sample.

In this case, any DFBETA with an absolute value greater than:

2 / sqrt(nrow(m1$model))

[1] 0.01628278

Let’s see how many of these we have:

table(abs(m1_inf$infmat[, 2] ) > (2/sqrt(nrow(m1$model))))


FALSE  TRUE 
14178   909

Now, for us, the DFBETAs become just like any other data point, and we can use a variety of EDA techniques to explore hypotheses about which observations represent these outlier points. Here’s one example, where we look at variables in the model and plot the DFBETAs against them – this shows the relationship between the value of ss1, it’s DFBETA, and how that relationship is different across reading and math.

plotdf <- sch_score_m1
plotdf$dfbeta_ss1 <- m1_inf$infmat[, 2]

ggplot(plotdf, aes(x = ss1, y = dfbeta_ss1)) + 
  geom_point(alpha=I(0.2)) + 
  geom_hline(yintercept = 2/sqrt(nrow(m1$model))) + 
  geom_hline(yintercept = -2/sqrt(nrow(m1$model))) + 
  facet_wrap(~subject) + 
  geom_rug(alpha=1/4, position = "jitter") +
  theme_bw()

This is one way to explore the DFBETAs, and what does it show us. First, it confirms that bigger and smaller values of the variable are more likely to have a bigger influence on the coefficient. But, the pattern is very different between the two subjects, and the distribution of outlier values (those outside of the horizontal lines) are also different. What does this evidence suggest about how you could change this model?

In this plot, we included a rug element on the bottom which shows each observed value of the x-axis and y-axis as a tick. Here we can see that though the distribution is balanced, extreme values are not balanced - with more extreme values present on the low-end of the scale (< -0.05) than at the high end (> 0.05). This means that there are more observations that are pulling the slope of ss1 downward to a high degree than upward. If we excluded all values of ss1 that are above and below the same cut-off, we would expect the coefficient on ss1 to be more positive in that case. Knowing the direction of this bias among the extreme values in our data is important in understanding how the model is behaving on this set of data.

0.3.1.2 Further Metrics

Two further metrics you may use to identify outlier observations are Cook’s D and DFITS. These produce a similar metric that combines the residuals and the leverage to measure true influence of each observation. Each is on a different scale, but should identify similar observations. Some rules of thumb for interpreting these metrics (from the excellent UCLA webbook on regression: https://stats.idre.ucla.edu/stata/webbooks/reg/chapter2/stata-webbooksregressionwith-statachapter-2-regression-diagnostics/)

Cook’s D ranges from 0 to very large positive values. The traditional cutoff point for high influence observations is 4/n where n is the number of rows in the sample.
DFITS can range from - large values to positive large values. The cutoff point for DFITS is 2 * sqrt(k/n) where k is the number of predictors in our model.

You can apply a similar analysis we just did for the DFBETAs to explore these. Let’s quickly go through an analysis where we look at the relationship between influential observations and variables that are not in our model. Let’s use DFITS because it keeps the directionality information (whether the influence is positive or negative).

plotdf <- full_data
plotdf$dffits <- dffits(m1)

# limits for dffits
dffits_limit <- 2 * sqrt(length(coef(m1)) / nrow(m1$model))

ggplot(plotdf) + aes(x = ss2, y = dffits) + 
    geom_point(alpha = 1/4) + 
    facet_grid(subject ~ grade) + 
    geom_hline(yintercept = dffits_limit) + 
    geom_hline(yintercept = -dffits_limit) + 
    theme_bw()

Here we have brought in grade level, a variable our original model left out. Now we see the motivation for splitting the model by sample.

Just for fun, let’s look at one more:

ggplot(plotdf) + aes(x = ss2, y = dffits, color = school_size) + 
    geom_point(alpha = 2/3) + 
    facet_wrap( ~locale_desc) + 
  scale_color_brewer(type = "qual", palette = 3, direction = -1) + 
  guides(color = guide_legend(override.aes = list(alpha = 1))) +
    geom_hline(yintercept = dffits_limit) + 
    geom_hline(yintercept = -dffits_limit) + 
    theme(legend.position = "bottom", 
                       legend.key.size = unit(12, "points"))

This type of analysis can give us insight into how our model should be modified or how our sample should be restricted to account for systematic variation in influential observations by key observable characteristics. Try some of your own analysis!

0.4 Regression Diagnostics

In the lecture we learned about the utility of regression diagnostic tests. Tests are useful because they can scale to multiple models and tell us which models to look into further and which models are OK. We can also make comparisons between sets of multiple model specifications by comparing how many of the fifty models fail the tests under different specifications. In short, model diagnostics are a scalable way to assess regression models.

While base-R provides some basic regression diagnostic functionality, it is more consistently implemented using the suite of functions provided by the lmtest package. These tests take some form of dividing the data or recasting the model, and then comparing for statistically meaningful differences in the divided data or in the recasted model residuals.

In general these functions generate a test statistic, degrees of freedom, and then generate a p-value against the null-hypothesis that the model is correctly specified. If the p-value is less than a critical value, the data supports the hypothesis that the model is not well-specified according to the particular test.

This is one of the rare times when I will tell you that the p-value is a useful statistic to consider on its own! It’s just more efficient if you want to test a model with multiple diagnostics, or multiple models with the same diagnostic. These tests have different test statistics and are all designed for interpretation around the p-value, so I recommend just using it. The caveat is that this is applied work and the diagnostics are intended to provide evidence of model failure. A model where a diagnostic p-value is below a critical value is a model that likely has a problem (though we can’t tell the degree of the problem from the diagnostic alone), while a model without a low p-value may or may not. Thus, this approach is a good first-pass, a noisy but useful indicator of how well specified these models may or may not be.

We’ve talked briefly about some of the tests in the lecture, but here we’ll review them in light of the options that can be passed to them. We’ll just focus on testing a single model for now, but at the bottom of this tutorial there is code for combining the tests and testing all the models together. For now, let’s focus on how the testing functions work in R.

0.4.1 Heteroskedasticity Tests

Breusch-Pagan Test, Golfeld-Quandt Test

The Breusch-Pagan test fits a linear model to the residuals of the regression model using the same predictors. If too much variance is explained by the predictors in the original model, then the model is considered to be heteroskedastic.

library(lmtest) # load the library for model testing

Loading required package: zoo

Attaching package: 㤼㸱zoo㤼㸲

The following objects are masked from 㤼㸱package:base㤼㸲:

    as.Date, as.Date.numeric

bptest(m1) # Breusch-Pagan test, takes a model object as its argument


    studentized Breusch-Pagan test

data:  m1
BP = 450.04, df = 3, p-value < 2.2e-16

This model fails the Breusch-Pagan test - there is evidence of heteroskedacticity given the model and the data.

The Goldfeld-Quandt test fits the original model to two subsets of the data defined by a breakpoint and compares the variances of the residuals. If the differences are statistically different, then the model is considered heteroskedastic. By default the sample is split in half, but because it involves splitting the data it is sensitive to the order of observations, and we can specify the order we which to place the observations in.

gqtest(m1) # model object


    Goldfeld-Quandt test

data:  m1
GQ = 1.0182, df1 = 7540, df2 = 7539, p-value = 0.2169
alternative hypothesis: variance increases from segment 1 to 2

gqtest(m1, 
       order.by = ~ss1, 
       data=m1$model, # one trick in R is if you want to pull the data a 
       # model was fit to, for a lm object you can use my_model$model to access it
       fraction = 0.4)


    Goldfeld-Quandt test

data:  m1
GQ = 0.52003, df1 = 4523, df2 = 4522, p-value = 1
alternative hypothesis: variance increases from segment 1 to 2

Here we see the challenge of regression diagnostics - the Breush-Pagan test said we had heteroskedacticity, and so do our residual plots, but the GQ test does not, even under different reorderings of the data.

This illustrates two issues to be wary of when using statistical diagnostics. First, all diagnostics/measures are imperfect and they should be weighed carefully with other forms of evidence. In this case, we are already skeptical of these models based on our analysis of the residuals and other factors so finding out that it may exhibit heteroskedacticity is not unexpected. Second, statistical diagnostics, like all other statistics, are susceptible to researchers “p-hacking” - running multiple tests with different parameters until a suitably low p-value is found. Try it with the gqtest() function - if you order the data the right way and set the fraction argument correctly, you will undoubtedly find a p-value below the critical value.

0.4.2 Autocorrelation Diagnostics

Durbin-Watson Test, Box-Pierce Test, Ljung-Box Test, Breusch-Godfrey Test

Autocorrelation is a problem with regression models where the residuals of one observation are correlated with the residuals of other observations in the data. Most commonly, autocorrelation occurs when observations are ordered by time and the observation of a unit in time t is correlated with its value at time t-1 (think the temperature day to day anywhere in the world except Montana).

Autocorrelation diagnostics, to detect this, rely on the ordering of the observations. They are especially important when modeling time-series data and including time as a feature in the model. Our model here does this implicitly modeling test scores in time t as a function of the scores in time t-1 and we have a test year variable in the data.

dwtest(ss2 ~ ss1, data = by_group$data[[6]])


    Durbin-Watson test

data:  ss2 ~ ss1
DW = 1.9884, p-value = 0.4541
alternative hypothesis: true autocorrelation is greater than 0

Box.test(residuals(lm(ss2~ss1, data = by_group$data[[6]])))


    Box-Pierce test

data:  residuals(lm(ss2 ~ ss1, data = by_group$data[[6]]))
X-squared = 0.0050397, df = 1, p-value = 0.9434

Box.test(residuals(lm(ss2~ss1, data = by_group$data[[6]])), type = "Ljung")


    Box-Ljung test

data:  residuals(lm(ss2 ~ ss1, data = by_group$data[[6]]))
X-squared = 0.0050764, df = 1, p-value = 0.9432

bgtest(ss2 ~ ss1, data = by_group$data[[6]])


    Breusch-Godfrey test for serial correlation of order up to 1

data:  ss2 ~ ss1
LM test = 0.00506, df = 1, p-value = 0.9433

Since we have already split our data out by test year, we have broken up the most obvious source of autocorrelation in our group-wise regression model strategy. Let’s look at some other potential ways we could imagine autocorrelation occurring.

However, more generally, autocorrelation can occur if there is some grouping of observations that isn’t accounted for – common in education. For example, in our data, the grade-subject-test_year school average observations are nested within districts and schools within the same district are more similar to one another than schools in different districts. Failing to account for this could cause autocorrelation in our models.

dwtest(ss2 ~ ss1, data = by_group$data[[6]], order.by = ~distid)


    Durbin-Watson test

data:  ss2 ~ ss1
DW = 1.7696, p-value = 0.008823
alternative hypothesis: true autocorrelation is greater than 0

bgtest(ss2 ~ ss1, data = by_group$data[[6]], order.by = ~distid)


    Breusch-Godfrey test for serial correlation of order up to 1

data:  ss2 ~ ss1
LM test = 5.1974, df = 1, p-value = 0.02262

When we look at it not temporally, but organizationally, we see evidence that there is indeed autocorrelation in our data – observations from the same district are correlated enough with one another to cause problems.

0.4.3 Multicollinearity

Aside from being the most straightforward critique of any social science statistical model, multicollinearity is a big problem for interpreting magnitude and statistical significance of individual predictors in a model. If two highly correlated variables are included in the model, then their coefficients will be a blend of the effect of both variables and the standard errors for these variables will be inflated and unreliable.

From UCLA’s Stata Regression diagnostic guide: > As a rule of thumb, a variable whose VIF values are greater than 10 may merit further investigation. It means that the variable could be considered as a linear combination of other independent variables.

# install.packages("car") if you don't already have the car package

# Fit a more complex model as an example
tmp_fit <- lm(ss2 ~ ss1 + factor(grade) * factor(test_year) + factor(subject) + n1, data = full_data)
car::vif(tmp_fit)

                                        GVIF Df GVIF^(1/(2*Df))
ss1                                 2.757167  1        1.660472
factor(grade)                     723.845760  4        2.277486
factor(test_year)                 141.459510  4        1.857073
factor(subject)                     1.007201  1        1.003594
n1                                  1.511774  1        1.229542
factor(grade):factor(test_year) 32802.831501 16        1.383956

0.4.4 Misspecification Diagnostics

Rainbow Test, RESET Test, Harvey-Collier Test

One form of ommitted variable biase we can easily test for is whether continuous measures in our model would be better described by polynomial terms instead of a single linear coefficient. Tests of this use different strategies.

The Rainbow test fits a regression to a middle portion of the data and does a test to see if it fits the ends of the data as well as the middle. If the fit for either tail is worse than for the center subsample, this is taken as evidence the model could be improved by polynomial terms.

raintest(ss2 ~ ss1, data = by_group$data[[6]])


    Rainbow test

data:  ss2 ~ ss1
Rain = 1.2376, df1 = 207, df2 = 204, p-value = 0.0638

The RESET test fits a model with a user-specified number of additional powers of the X variables and then does a standard F-test to see if this new model outperforms the original model. The most common test is the Ramsey Regression Equation Specification Error Test, or RESET, which checks whether there are omitted polynomial terms to a predictor that would improve model fit.

This tells us that the sum of squares will be meaningfully reduced with the simple inclusion of polynomials of the main predictor – ss1. Thus the new model should take the form:

\[SS_{2} = \alpha + \beta_{1}SS_{1} + \beta_{2}SS_{1}^2 + \beta_{3}SS_{1}^3 + \beta_{4}SS_{1}^4\]

reset(ss2 ~ ss1, power= 2:4, data = by_group$data[[6]])


    RESET test

data:  ss2 ~ ss1
RESET = 3.3893, df1 = 3, df2 = 408, p-value = 0.0181

The Harvey-Collier test conducts a t-test on the means of the recursive residuals. The means should not differ from 0 significantly, so if they do, this is taken as evidence supporting the need for polynomial terms.

harvtest(ss2 ~ ss1, data = by_group$data[[6]])


    Harvey-Collier test

data:  ss2 ~ ss1
HC = 0.81994, df = 410, p-value = 0.4127

These can also optionally be ordered by a vector that may be meaningful such as a grouping term or the value of a predictor. We might consider ordering the data by class size (n1) or pre-test score (ss1) or school district distid.

raintest(ss2 ~ ss1, data = by_group$data[[6]], order.by = ~ss1)


    Rainbow test

data:  ss2 ~ ss1
Rain = 1.0454, df1 = 207, df2 = 204, p-value = 0.3753

harvtest(ss2 ~ ss1, data = by_group$data[[6]], order.by = ~ss1)


    Harvey-Collier test

data:  ss2 ~ ss1
HC = 2.7764, df = 410, p-value = 0.005749

The same concerns about p-hacking apply.

0.5 Model Comparisons

After we have diagnosed the model, we want to propose alternative models, and we need a framework for assessing the difference between any alternatives and our existing model.

Remember that in our previous work we not only identified alternative models using the additional data we’ve merged in, but we even found improved model fit by including polynomial regression terms to account for the non-linear relationship we observed between ss1 and ss2 at the tails of the distribution.

There are multiple ways to compare models depending on the purposes of the model and the goals of the analyst. We will discuss three here:

Comparing model fit to the data
Comparing model improvement with F-tests
Comparing model predictive power

In this section, we’ll look at how we can do model comparison across our fifty models. This will allow us to compare the same model across different subsets of data (within the fifty models) and compare new model fits to the same data (compare model A to model B across each of the fifty sets of data). Both of these strategies are helpful when evaluating models repeated across subsets.

First, let’s look at a method we can use to get and plot the model fits for each of our 50 submodels using the glance function to extract the model fit statistics from each group.

glance <- by_group %>%
  mutate(glance = map(model, broom::glance)) %>%
  unnest(glance, .drop = TRUE)

arrange(glance, desc(r.squared))



ggplot(glance, aes(x = grade, y = r.squared, color = subject)) +
  geom_jitter(width = 0.1, height = 0) + theme_bw()

This plot shows 50 points, one for each combination of grade, year, and subject. Each point represents the r-squared of our proposed model (ss2 ~ ss1) fit to that specific subset of the data. The plot shows a clear pattern, our model fits better in higher grades than in lower grades.

Before we go comparing alternatives to all fifty models, let’s look at the three ways we can compare models through the lens of a a single year-grade-subject model. We’ll start by adding the class size, one of the options we explored in the previous guided tutorial, to build an alternative:

m1 <- by_group$model[[15]]
m2 <- lm(ss2 ~ ss1 + n1, data = by_group$data[[15]])
summary(m1)


Call:
lm(formula = ss2 ~ ss1, data = df)

Residuals:
     Min       1Q   Median       3Q      Max 
-108.659  -10.152    1.222   10.895   77.899 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) -129.49595   28.39734   -4.56  6.7e-06 ***
ss1            1.16395    0.02665   43.67  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 19.95 on 425 degrees of freedom
Multiple R-squared:  0.8177,    Adjusted R-squared:  0.8173 
F-statistic:  1907 on 1 and 425 DF,  p-value: < 2.2e-16

summary(m2)


Call:
lm(formula = ss2 ~ ss1 + n1, data = by_group$data[[15]])

Residuals:
     Min       1Q   Median       3Q      Max 
-104.775  -10.628    1.521   11.043   82.040 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) -106.20550   29.00342  -3.662 0.000282 ***
ss1            1.13554    0.02780  40.847  < 2e-16 ***
n1             0.12817    0.03977   3.223 0.001366 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 19.73 on 424 degrees of freedom
Multiple R-squared:  0.8221,    Adjusted R-squared:  0.8213 
F-statistic: 979.7 on 2 and 424 DF,  p-value: < 2.2e-16

Let’s look at the “Multiple R-squared” measure here. For the first model, it is 0.8177458 and for the second model it is 0.8221043. This is a difference of 0.005. While there are no hard decision rules for how much improvement in r-squared is meaningful or a significant improvement (more on this later), we can be confident that this is not it.

However, it can be helpful to formally measure these things. We can use a formal F-test to directly compare these two models since the original model is fully nested within the new model – they are identical except for the addition of one predictor and fit to the same sample. An F-test gives us a p-value to tell us whether the improvement in the fit to the data between the two models was a statistically significant difference or not.

Confusingly, the R command to conduct this F-test is called anova()

anova(m1, m2)

Analysis of Variance Table

Model 1: ss2 ~ ss1
Model 2: ss2 ~ ss1 + n1
  Res.Df    RSS Df Sum of Sq      F   Pr(>F)   
1    425 169107                                
2    424 165063  1    4044.1 10.388 0.001366 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

So even with a very modest improvement in R-squared, the F-test tells us that the data support the conclusion that model 2 is a better fit to the data than model 1. This is a case where we need to consider our question and our goals to determine whether the model is superior or not.

F-tests are really useful because they allow us to test the joint significance of multiple variables. We can also test multiple nested models at once. Here, let’s add a set of polynomial terms and jointly test whether as a group they provide a statistically significant better model fit.

m3 <- lm(ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1, 
         data = by_group$data[[15]])
anova(m1, m2, m3)

Analysis of Variance Table

Model 1: ss2 ~ ss1
Model 2: ss2 ~ ss1 + n1
Model 3: ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1
  Res.Df    RSS Df Sum of Sq       F  Pr(>F)   
1    425 169107                                
2    424 165063  1    4044.1 10.5269 0.00127 **
3    421 161735  3    3328.0  2.8876 0.03535 * 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

# summary(m3)

Here, we see that the polynomial terms again provide a statistically significant improvement to fit, even over our model with cohort size.

summary(m3)


Call:
lm(formula = ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1, 
    data = by_group$data[[15]])

Residuals:
    Min      1Q  Median      3Q     Max 
-98.324 -10.747   0.863  11.323  81.035 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept)  4.396e+05  1.730e+05   2.542 0.011391 *  
ss1         -1.687e+03  6.667e+02  -2.530 0.011770 *  
I(ss1^2)     2.426e+00  9.628e-01   2.520 0.012103 *  
I(ss1^3)    -1.548e-03  6.173e-04  -2.507 0.012541 *  
I(ss1^4)     3.698e-07  1.483e-07   2.494 0.013029 *  
n1           1.362e-01  4.025e-02   3.384 0.000782 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 19.6 on 421 degrees of freedom
Multiple R-squared:  0.8257,    Adjusted R-squared:  0.8236 
F-statistic: 398.9 on 5 and 421 DF,  p-value: < 2.2e-16

We see that our R-squared has increased and each individual variable is statistically significant, so the F-test makes some sense. Using F-tests is a useful way to understand the impact additional predictors have on our model, so it’s a good tool to have in your toolbelt.

Let’s look at how we can apply this F-test approach to models across different subsets in our example problem here.

# Define a grouped dataframe using only the columns we need (1:10)
# group by the grade, subject, and year
# Then nest so the data hangs together in one list
by_group <- full_data[, 1:10] %>%
  group_by(grade, subject, test_year) %>%
  nest()

# Define a simple function that fits our model
simple_model <- function(df) {
  lm(ss2 ~ ss1, data = df)
}

alt_model <- function(df) {
  lm(ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1, data = df)
}

# Apply that model to each group in our dataset using the `map` function
# This is equivalent to a loop, but more efficient to write
by_group <- by_group %>%
  mutate(base_model = map(data, simple_model), 
         full_model = map(data, alt_model))

# Apply a function that makes an F-test of our two models (x and y)
# returns only the statistics from the anova object we want
# in this case, the difference in the sum of squares and the p-value of the 
# F-test
simple_anova <- function(x, y, stat = c("ss", "p"), ...){
  out <- anova(x, y)
  if(stat == "ss"){
    return(out$`Sum of Sq`[[2]])
  } else if(stat == "p"){
    return(out$`Pr(>F)`[[2]])
  }
}

ftests <- by_group %>% rowwise() %>%
    do(ss = unlist(simple_anova(x = .$base_model, y = .$full_model, stat = "ss")), 
      pval = unlist(simple_anova(x = .$base_model, y = .$full_model, stat = "p")))

ftests <- apply(ftests, 2, unlist) # convert to a numeric object
ftests <- as.data.frame(ftests) # make into a data.frame for ease of use
table(ftests$pval < 0.05)


FALSE  TRUE 
   21    29

This table is telling us that in roughly half of the cases, the data support the conclusion that the fuller model better describes the data.

As we saw, the r-squared improvement was marginal, but the F-test improvement was statistically significant. These are answering slightly different questions, because the F-test is intended to specifically take into account the fact that every additional variable added improves model fit to the data, so the improvement must be big enough to be better than this. The adjusted R-squared attempts to do the same thing, but we don’t have a sense of what a meaningful increase in the adjusted R-squared from this number alone, so we are left to eyeball it and use our judgment – which is acceptable and good enough in many cases.

Both of these measures are only telling us how well our model fits this data, though. That is, how well does our model fit this specific sample of data that it has been shown. Often in applied analysis, we want to find a model that will fit this year’s data, but also fit next year’s data as well. In this case, we are interested in the predictive power of our model on new data.

There are model comparison methods that allow us to do just that by looking at how well our model predicts the data. We’ll focus on the root mean squared error (RMSE) - one of the most common of these metrics. The RMSE is the square root of the mean of the squared errors. So to calculate it we take our residuals, square them, divide by the number of residuals (to get the mean of the squares), and then take the square root of this value. It provides a good balance between being sensitive to large errors, and rewarding models that fit the whole dataset well. Because it is measuring the error, a lower number is better. 0 is the theoretical minimum.

You can find many alternatives like the Median Absolute Error (MAE), Mean Absolute Deviation (MAD), and many more. But, RMSE is fairly standard and implemented in most software.

Measuring predictive performance of models its own full guide, but here are two main points:

Always measure performance on data the model has not “seen” (e.g. was not used to fit the model). Models overfit to the data they are given, so they appear more accurate than they will be on future data.
Measure the performance and variability in performance where possible. We want to know not just how accurate the model is, but how precisely we can measure that accuracy.

To achieve this goal we use a technique called resampling. There are many forms of resampling – bootstrapping, crossvalidation, leave-one-out, and others. You can find excellent treatments of these concepts online. Each of them is a method to divide our data up into data the model sees (training data) and data we use to measure the performance of the model (test data). We fit the model to the training data, evaluate it on the test data, and repeat. We will discuss this more at September Workshop.

To give you a sense of this technique now, let’s work a simple example where we extract the RMSE for one of the fifty models and estimate it from resampled ata.

For now, we will use a technique called
k-fold cross-validation because it is easy to implement and a standard. This method “folds” the data K (here k = 5) times to create K distinct test sets (each consisting of 1/kth of the data). Then the model is fit K times, to the data that is not in each kth test set, and evaluated on that test set. This gives us K estimates of the model fit from our original sample. In this case, we start with 251 observations in our dataset, fold it 5 times, and the result is 5 training sets with ~200 observations and 5 corresponding test sets with ~50 observations to measure the model performance against (using RMSE in this case).

cv_dat <- crossv_kfold(by_group$data[[7]], k = 5)
models1 <- map(cv_dat$train, ~lm(ss2 ~ ss1, data = .))
models2 <- map(cv_dat$train, ~lm(ss2 ~ ss1 + n1, data = .))
models3 <- map(cv_dat$train, ~lm(ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1, 
                                 data = .))

plotdf <- data.frame(model_1 = map2_dbl(models1, cv_dat$test, rmse), 
                     model_2 = map2_dbl(models2, cv_dat$test, rmse),
                     model_3 = map2_dbl(models3, cv_dat$test, rmse))
plotdf

The downside of k-fold cross-validation is that it uses a lot of data – we can only generate a few unique training/test sets (in this case 5) when we only have 250 observations. This makes it hard to draw conclusions about the difference between our models - we don’t have enough data. Look at the results, it looks like model 3 generally has a lower (better) RMSE.

As an alternative, for cases where we do not have a lot of data to generate enough holdout sets, we can use a more general technique called bootstrapping. In this case, we fit our model to the full sample - so we get the benefit of all the observations. Then, we evaluate its performance against N resamples created by sampling the data with replacement. So, in this case, we will create 100 resamples of the same size as the full sample, drawn from the full sample but drawn with replacement. This will give us 100 estimates of the RMSE instead of 5.

boot_dat <- bootstrap(by_group$data[[7]], n = 100) # bootstrap samples
models1 <- lm(ss2 ~ ss1, data = by_group$data[[7]]) # fit model 1 to original sample
models2 <- lm(ss2 ~ ss1 + n1, data = by_group$data[[7]])
models3 <- lm(ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1, 
              data = by_group$data[[7]])


plotdf <- data.frame(model_1 = map_dbl(boot_dat$strap, 
                                       ~rmse(models1, data = .x)), # calculate RMSE
                     # on each subsample
                     model_2 = map_dbl(boot_dat$strap, ~rmse(models2, data = .x)),
                     model_3 = map_dbl(boot_dat$strap, ~rmse(models3, data = .x)))

summary(plotdf)

    model_1         model_2         model_3     
 Min.   :15.51   Min.   :15.20   Min.   :14.79  
 1st Qu.:17.55   1st Qu.:17.52   1st Qu.:16.72  
 Median :18.21   Median :18.10   Median :17.30  
 Mean   :18.33   Mean   :18.21   Mean   :17.38  
 3rd Qu.:19.00   3rd Qu.:18.86   3rd Qu.:18.05  
 Max.   :21.01   Max.   :20.87   Max.   :20.63

With 100 measures of RMSE it looks clear that our third model is superior - it has a lower average RMSE and it has a lower maximum and minimum values of RMSE as well. You can use EDA techniques to plot these differences as well if you like.

This brief introduction has demonstrated three possible ways you can conduct model comparison:

Informally, how well does each model explain the variance in the data it is fit to (R-squared)
Directly and formally, does the data support the evidence that one model fits the data better than another model (F-tests)
Directly and informally, does one model have a lower predictive error on data it has not seen before than the other model (resampling + RMSE)

We will develop and extend these ideas of predictive performance of our models in future workshops and tutorials.

0.6 Communicating Regression Models with Graphics

As discussed in the lecture, interpretation of regression models can be very difficult as models move beyond simple bi-variate relationships. We can look at coefficients and model fit statistics, but this is like looking at the gears of a machine and trying to picture how it works. It’s easier if we just turn it on! And regression models are computational machines designed to make predictions, so we can use that feature to simulate values for them and watch how they behave.

Fortunately, this powerful technique builds on skills we have already developed for exploratory data analysis. The only difference is that here the data we are exploring are how our model predicts outcomes in response to different input we give it.

Let’s look at some examples.

First, let’s create a coefficient plot and gaze upon the gears of our model:

library(coefplot) # easy package for visualizing lm coefficients
coefplot(m3, intercept = FALSE) +  # intercept = FALSE avoids skewing the plot
  # because of the large magnitude of the intercept
  theme_bw()

This graphic is not very enlightening. Do you know why? If we wrote out the equation for our regression model, we would see that because of the scale of our predictor variables (and the ss1 predictor specifically) and our outcome, the coefficients have to have very large and very small values to fit the data well when incorporating polynomial terms. Using polynomial terms results in making our coefficients much harder to interpret because of the magnitudes of variables involved.

coefplot(m2, intercept = FALSE) + theme_bw()

Here’s a simpler plot corresponding to a simpler model.

But we don’t want to be limited in our interpretation to simple models, so let’s look at techniques that allow us to interpret and understand more complex models and go beyond the traditional reporting of coefficients.

0.6.1 Margin Plots

Margin plots are graphics that look at how the predictor or the coefficient of interest behaves for certain subsets of the data. This helps us look at how the model behavior changes as values of key variables in our data change. This is a powerful technique useful for exploring complex model behaviors. Let’s return to our polynomial model:

library(margins) # package to compute marginal plots
margins(m3)

Average marginal effects
lm(formula = ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4) + n1,     data = by_group$data[[15]])

marginal_effects(m3)


# Spurious
margins::cplot(m3, "ss1")

NA

This clearly shows us the polynomial shape that the additional terms are letting us explore. Let’s look at another example - this time let’s look at how we can use marginal plots to explore a model that includes categorical variables and an interaction between those variables and the outcome.

m4 <- lm(ss2 ~ ss1 + ss1 * locale_desc, data = full_data)
coefplot(m4, intercept = FALSE) + theme_bw()

margins::cplot(m4, "ss1") # average effect of ss1

margins::cplot(m4, "locale_desc") # average effect of locale

margins::cplot(m4, "locale_desc", dx = "ss1", what = "effect") # keep

The above figure shows the precision and variability in estimates of ss1 for each locale, based on the interaction term. These effects are plotted on the scale of the ss1 coefficient and represent the sum of the ss1 coefficient and the interaction effect - this is a more direct way to interpret interaction effects than the coefplot or table of coefficients.

We can also plot multiple lines, one for each locale, to look at how the line is changed by each locale. This isn’t the only way to plot multiple lines from this model, but you can do the computation with the margin package and add the lines together on the same plot with this code:

local({
  cplot(m4, "ss1", data = full_data[full_data$locale_desc == "Rural, outside MSA",])
  cplot(m4, "ss1", data = full_data[full_data$locale_desc == "Mid-Size City",], 
        draw = "add", col = "blue")
  cplot(m4, "ss1", data = full_data[full_data$locale_desc == "Small Town",], 
        draw = "add", col = "red")
  cplot(m4, "ss1", data = full_data[full_data$locale_desc == "Urban Fringe of a Large City",], 
        draw = "add", col = "purple")
  cplot(m4, "ss1", data = full_data[full_data$locale_desc == "Mid-Size City",], 
        draw = "add", col = "gold")
  cplot(m4, "ss1", data = full_data[full_data$locale_desc == "Large City",], 
        draw = "add", col = "orange")
})

NA

The advantage of marginal plots is they allow you to interpret the model on the scale of the outcome, not on the scale of the coefficients. This can be much more intuitive to understand and allow you to present to audiences that are familiar with the outcome variable, but not necessarily the inner-workings of regression models.

0.6.2 Counterfactual Plots

A final visual way to explore our data is to create counterfactuals and see how reference observations or ranges of predictors change across other values. Let’s continue with looking at locale based on our belief that different geographic locations have different growth functions on test scores – perhaps due to the local teaching labor market or some other community factors.

Luckily, counterfactual plots are easy to make. We just have to get comfortable making fake data!

So let’s look at how the predicted score of an observation would change based on it being assigned to a different locale. This is a very hypothetical counterfactual, since we cannot move schools from one geographic region to another, but it is an important one for highlighting how our model views the data.

m5 <- lm(update(formula(m3), . ~ . + ss1 * locale_desc), data = full_data)
# Pick a row, here we sample at random
example_school <- full_data[sample(row.names(full_data), 1), ]
example_school <- example_school[rep(seq_len(nrow(example_school)),
                   each=length(unique(full_data$locale_desc))),]
example_school$locale_desc <- unique(full_data$locale_desc)
example_school$yhat <- predict(m5, newdata = example_school)

ggplot(example_school, aes(x = locale_desc, y = yhat)) + 
  geom_hline(yintercept = example_school$ss2[1], color = I("red")) +
  geom_point() + theme_bw() + 
  theme(axis.text = element_text(angle = 25))

NA

From this plot we can see that the observed value (red horizontal line) deviates than any prediction no matter the locale we are in. We can also see that the model believes our school would fare poorly in a Large City, but do best if it was rural, but inside an MSA.

0.7 Robust and Weighted Regression

One problem we have noted with our data previously is that it treats big and small schools alike. For some analyses this is fine, but for others, it may cause our analysis problems. In these cases, we can weight our model by the number of students in a school so that observations with large numbers of schools will be “counted more” by the regression models than observations with fewer schools.

Comparing weighted and unweighted models is not something that can be done with an F-test. However, by using predictive model fit metrics like the RMSE this is possible. Let’s see how our polynomial model changes when we weight by the number of students in a cohort.

unweighted_mod <- lm(ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4), 
                          data = by_group$data[[7]])
weighted_mod <- lm(ss2 ~ ss1 + I(ss1^2) + I(ss1^3) + I(ss1^4), 
                          data = by_group$data[[7]], weights = n1)


plotdf <- data.frame(unweight_rmse = map_dbl(boot_dat$strap, 
                                       ~rmse(unweighted_mod, data = .x)), 
                     weight_rmse = map_dbl(boot_dat$strap,
                                           ~rmse(weighted_mod, data = .x)))
                
ggplot(plotdf, aes(x = unweight_rmse)) + 
  geom_density(color = I("purple")) + 
  geom_density(aes(x = weight_rmse), color = I("darkgreen")) + 
  theme_bw() + labs(x = "RMSE")

The weighted RMSE (dark green) has a higher distribution over our bootstrap resamples than the unweighted RMSE (purple). Remember that we can interpret the RMSE as being on the scale of the square root of the outcome. Our outcome has a mean of ~1100 (square root ~33) . Our average predicted observation error then represents a substantial fraction of the mean of our outcome.

Compare two example models to get a sense of how much the weights change things.

margins::cplot(weighted_mod, "ss1", col = "darkgreen")

margins::cplot(unweighted_mod, "ss1", draw = "add", col = "purple")

The coefficients change a lot between these two models, just by modifying the weights. But, when we plot these models we can see that they are still largely the same on the scale of the outcome.

0.7.1 Robust Regression

Those of you who have used Stata may be wondering where robust regression estimates come into play. Robust regression comes in two forms: the first is robust regression - a regression technique that results in different coefficients and different standard errors. Robust regression is designed to be more resistant to the influence of extreme outlier observations, so it can be a useful technique for analyzing data with outliers. It achieves this by using a different method for calculating the least-squares equation. This makes it different from the second form of robustness, a correction of the standard errors.

In practice, the change in coefficients from robust regression will often be small. For our example here, we will fit our model to the full data set, not one of our fifty subsamples.

source("R/robust_functions.R")
library(MASS)


Attaching package: 㤼㸱MASS㤼㸲

The following object is masked from 㤼㸱package:dplyr㤼㸲:

    select

library(sandwich)
library(lmtest)

full_data$frl_per[is.na(full_data$frl_per)] <- 0
m_notrobust <- lm(ss2 ~ ss1 + n1 + total_enrollment + frl_per + att_rate + 
                    factor(grade) + subject + factor(test_year), 
                  data = full_data)
m_robust <- rlm(ss2 ~ ss1 + n1 + total_enrollment + frl_per + att_rate + 
                  factor(grade) + subject + factor(test_year), 
                  data = full_data)

coefplot::multiplot(m_robust, m_notrobust, intercept = FALSE) + theme_bw()

We do not see dramatically different results between our robust and non-robust estimates. You can confirm this by comparing the coefficients and standard errors directly using the summary function for each model.

summary(m_notrobust)


Call:
lm(formula = ss2 ~ ss1 + n1 + total_enrollment + frl_per + att_rate + 
    factor(grade) + subject + factor(test_year), data = full_data)

Residuals:
    Min      1Q  Median      3Q     Max 
-133.71  -12.34   -0.04   12.27  149.36 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)    
(Intercept)           312.132086   7.825892  39.885  < 2e-16 ***
ss1                     0.697045   0.005028 138.625  < 2e-16 ***
n1                      0.024466   0.004435   5.517 3.50e-08 ***
total_enrollment       -0.007228   0.001279  -5.652 1.62e-08 ***
frl_per                -0.477890   0.010332 -46.255  < 2e-16 ***
att_rate                1.053568   0.067633  15.578  < 2e-16 ***
factor(grade)5         -5.883703   0.548986 -10.717  < 2e-16 ***
factor(grade)6         17.544692   0.759360  23.105  < 2e-16 ***
factor(grade)7         21.496432   0.895261  24.011  < 2e-16 ***
factor(grade)8         18.608891   1.042392  17.852  < 2e-16 ***
subjectread           -25.115339   0.347782 -72.216  < 2e-16 ***
factor(test_year)2008  -4.821290   0.540760  -8.916  < 2e-16 ***
factor(test_year)2009   1.502878   0.545662   2.754  0.00589 ** 
factor(test_year)2010   1.009739   0.555442   1.818  0.06910 .  
factor(test_year)2011   2.880695   0.561161   5.133 2.88e-07 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 21.2 on 15072 degrees of freedom
Multiple R-squared:  0.926, Adjusted R-squared:  0.926 
F-statistic: 1.348e+04 on 14 and 15072 DF,  p-value: < 2.2e-16

summary(m_robust)


Call: rlm(formula = ss2 ~ ss1 + n1 + total_enrollment + frl_per + att_rate + 
    factor(grade) + subject + factor(test_year), data = full_data)
Residuals:
       Min         1Q     Median         3Q        Max 
-134.21835  -12.30843    0.01496   12.31326  149.69479 

Coefficients:
                      Value    Std. Error t value 
(Intercept)           298.7856   7.1983    41.5080
ss1                     0.6908   0.0046   149.3602
n1                      0.0207   0.0041     5.0643
total_enrollment       -0.0067   0.0012    -5.6943
frl_per                -0.4651   0.0095   -48.9378
att_rate                1.2463   0.0622    20.0335
factor(grade)5         -5.6955   0.5050   -11.2791
factor(grade)6         19.4666   0.6985    27.8708
factor(grade)7         22.7166   0.8235    27.5867
factor(grade)8         20.6373   0.9588    21.5242
subjectread           -24.3884   0.3199   -76.2398
factor(test_year)2008  -3.8996   0.4974    -7.8401
factor(test_year)2009   1.6218   0.5019     3.2313
factor(test_year)2010   1.4430   0.5109     2.8244
factor(test_year)2011   3.3372   0.5162     6.4655

Residual standard error: 18.25 on 15072 degrees of freedom

One of the problems we were looking to solve is the distribution of our errors. We can run a model check to see if the residuals from our robust model are better.

plotdf <- data.frame(y = m_robust$model$ss2, 
                     pred_robust = predict(m_robust, data = m_robust$model), 
                     pred_norobust = predict(m_notrobust, data = m_robust$model))
plotdf$resid_robust <- plotdf$y - plotdf$pred_robust
plotdf$resid_norobust <- plotdf$y - plotdf$pred_norobust

ggplot(plotdf) + 
  geom_point(alpha = I(0.1), aes(x = pred_norobust, y = resid_norobust)) + 
  geom_hline(yintercept = 0, linetype = 2) +
  geom_smooth(aes(x = pred_robust, y = resid_robust), 
              color = I("red"), se = FALSE) + 
  geom_smooth(aes(x = pred_norobust, y = resid_norobust), 
              color = I("black"), se=FALSE) + 
  
  theme_bw()

The robust estimator does almost nothing to improve our residual plot either - even at extreme values the attenuation of the residuals is small. So there are limits to the strength of robust regression aiding in the presence of outliers.

However, the robust estimates of our parameters, coefficients, should be less influenced by these outliers. So robustness does not help us with prediction very much, but it does help us adjust for the bias that can occur in our coefficients when there are strong outliers in the data. This is why it is important to understand the purpose of the models we are fitting, in order to understand when different corrections are appropriate. We should prefer the robust coefficients for understanding the relationships within our data, but we may not prefer the robust coefficients for the purposes of making new predictions on future data.

0.7.1.1 Robust Standard Errors

In addition to robust regression estimates, a more standard procedure is to adjust the standard errors in our model with a postestimation adjustment. In this case, the model coefficients stay the same - they are the OLS betas - but the standard errors have been adjusted to reflect the non-indepedence between our observations.

In R this procedure is much more complicated than Stata, owing to R’s refusal to implement defaults. There are many different acceptable estimators to adjust the standard errors - Stata uses a default and does not bother the user. R refuses. To replicate the behavior of Stata, use “HC1” as the adjustment.

library(sandwich)
library(lmtest)
summary(m_notrobust)$coefficients

                           Estimate  Std. Error    t value      Pr(>|t|)
(Intercept)           312.132085622 7.825891634  39.884540  0.000000e+00
ss1                     0.697044946 0.005028281 138.624901  0.000000e+00
n1                      0.024466246 0.004434683   5.517022  3.504733e-08
total_enrollment       -0.007228255 0.001278943  -5.651740  1.617077e-08
frl_per                -0.477890499 0.010331566 -46.255377  0.000000e+00
att_rate                1.053568410 0.067632546  15.577831  2.728769e-54
factor(grade)5         -5.883703424 0.548985548 -10.717410  1.052428e-26
factor(grade)6         17.544691785 0.759359545  23.104591 4.291026e-116
factor(grade)7         21.496431912 0.895261144  24.011354 4.665553e-125
factor(grade)8         18.608890960 1.042391851  17.852107  1.481198e-70
subjectread           -25.115338571 0.347782337 -72.215682  0.000000e+00
factor(test_year)2008  -4.821289740 0.540759877  -8.915768  5.391878e-19
factor(test_year)2009   1.502877854 0.545661523   2.754231  5.890077e-03
factor(test_year)2010   1.009739161 0.555442150   1.817902  6.909902e-02
factor(test_year)2011   2.880695334 0.561161117   5.133455  2.880108e-07

coeftest(m_notrobust, 
         vcov = vcovHC(m_notrobust, "HC1"))


t test of coefficients:

                         Estimate  Std. Error  t value  Pr(>|t|)    
(Intercept)           312.1320856  17.4992766  17.8369 < 2.2e-16 ***
ss1                     0.6970449   0.0067522 103.2321 < 2.2e-16 ***
n1                      0.0244662   0.0042171   5.8017 6.697e-09 ***
total_enrollment       -0.0072283   0.0012816  -5.6400 1.731e-08 ***
frl_per                -0.4778905   0.0149445 -31.9777 < 2.2e-16 ***
att_rate                1.0535684   0.1867300   5.6422 1.709e-08 ***
factor(grade)5         -5.8837034   0.6090303  -9.6608 < 2.2e-16 ***
factor(grade)6         17.5446918   0.8857173  19.8085 < 2.2e-16 ***
factor(grade)7         21.4964319   1.1107137  19.3537 < 2.2e-16 ***
factor(grade)8         18.6088910   1.3802557  13.4822 < 2.2e-16 ***
subjectread           -25.1153386   0.3466552 -72.4505 < 2.2e-16 ***
factor(test_year)2008  -4.8212897   0.5405905  -8.9186 < 2.2e-16 ***
factor(test_year)2009   1.5028779   0.5287546   2.8423  0.004485 ** 
factor(test_year)2010   1.0097392   0.5478816   1.8430  0.065350 .  
factor(test_year)2011   2.8806953   0.5723193   5.0334 4.875e-07 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Remember, this adjustment only affects the standard errors - the coefficients remain the same. (If this code doesn’t work, make sure to install gridExtra first)

gridExtra::grid.arrange(coefplot(m_notrobust, intercept = FALSE, 
                                 sort = "alphabetical") + theme_bw(), 
                        coefplot_coeftest(
                          coeftest(m_notrobust, vcov = vcovHC(m_notrobust, "HC1")), 
                          intercept = FALSE)
                        )

In this case, the adjustment looks minor - it’s nearly impossible to distinguish in our coefficient plots and does not affect our decision about the statistical significance of any of our predictors. This is a good sign!

0.7.1.2 Cluster Standard Errors

There is a special case of robust standard errors that are very important in education regression modeling - cluster standard errors. Cluster standard errors allow us to account for the fact that observations drawn from the same “cluster” covary with each other, and thus they are not fully independent. We can use this information to get a more accurate estimate of our model’s certainty - generally by inflating the standard errors a bit to account for this duplicated information.

This technique is important in education because education data is full of “clusters” - repeated test scores for the same student, students in the same classroom, classrooms in the same school, schools in the same district, and even districts in the same state.

Let’s just look at one example where we cluster on the school district code in our data.

cluster_vcov <- get_CL_vcov(m_notrobust, full_data$distid)
summary(m_notrobust)$coefficients

                           Estimate  Std. Error    t value      Pr(>|t|)
(Intercept)           312.132085622 7.825891634  39.884540  0.000000e+00
ss1                     0.697044946 0.005028281 138.624901  0.000000e+00
n1                      0.024466246 0.004434683   5.517022  3.504733e-08
total_enrollment       -0.007228255 0.001278943  -5.651740  1.617077e-08
frl_per                -0.477890499 0.010331566 -46.255377  0.000000e+00
att_rate                1.053568410 0.067632546  15.577831  2.728769e-54
factor(grade)5         -5.883703424 0.548985548 -10.717410  1.052428e-26
factor(grade)6         17.544691785 0.759359545  23.104591 4.291026e-116
factor(grade)7         21.496431912 0.895261144  24.011354 4.665553e-125
factor(grade)8         18.608890960 1.042391851  17.852107  1.481198e-70
subjectread           -25.115338571 0.347782337 -72.215682  0.000000e+00
factor(test_year)2008  -4.821289740 0.540759877  -8.915768  5.391878e-19
factor(test_year)2009   1.502877854 0.545661523   2.754231  5.890077e-03
factor(test_year)2010   1.009739161 0.555442150   1.817902  6.909902e-02
factor(test_year)2011   2.880695334 0.561161117   5.133455  2.880108e-07

coeftest(m_notrobust, vcov = cluster_vcov)


t test of coefficients:

                         Estimate  Std. Error  t value  Pr(>|t|)    
(Intercept)           312.1320856  44.3524147   7.0375 2.041e-12 ***
ss1                     0.6970449   0.0116012  60.0837 < 2.2e-16 ***
n1                      0.0244662   0.0058884   4.1550 3.271e-05 ***
total_enrollment       -0.0072283   0.0042111  -1.7165  0.086098 .  
frl_per                -0.4778905   0.0224800 -21.2585 < 2.2e-16 ***
att_rate                1.0535684   0.5577419   1.8890  0.058912 .  
factor(grade)5         -5.8837034   0.7235308  -8.1319 4.552e-16 ***
factor(grade)6         17.5446918   1.9645087   8.9308 < 2.2e-16 ***
factor(grade)7         21.4964319   1.7423489  12.3376 < 2.2e-16 ***
factor(grade)8         18.6088910   2.0063794   9.2749 < 2.2e-16 ***
subjectread           -25.1153386   0.4858407 -51.6946 < 2.2e-16 ***
factor(test_year)2008  -4.8212897   1.8661521  -2.5835  0.009788 ** 
factor(test_year)2009   1.5028779   0.6890931   2.1810  0.029202 *  
factor(test_year)2010   1.0097392   1.0912327   0.9253  0.354814    
factor(test_year)2011   2.8806953   1.7354961   1.6599  0.096962 .  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

coefplot(m_notrobust, intercept = FALSE)

coefplot_coeftest(coeftest(m_notrobust, vcov = cluster_vcov), intercept = FALSE)

We see that clustering has a much bigger impact on our standard errors than the robustness correction we applied above. Here we see the standard errors increase by 2-5x in size, resulting in the att_rate predictor that once seemed statistically significant to no longer be so.

Clustering in this way does not help with our residuals though because while it changes the standard errors around our coefficients and predictions, it does not change the coefficients which are used to generate the prediction. See for yourself:

plotdf <- data.frame(y = full_data$ss2, 
                     yhat_nr = fitted(m_notrobust), 
                     yhat_r = predict.robust(m_notrobust, data = full_data, 
                                             robust_vcov = cluster_vcov))

plotdf$resid_robust <- plotdf$y - plotdf$yhat_r.fit.fit
plotdf$resid_norobust <- plotdf$y - plotdf$yhat_nr

ggplot(plotdf) + 
  geom_point(alpha = I(0.1), aes(x = yhat_nr, y = resid_norobust)) + 
  geom_smooth(aes(x = yhat_r.fit.fit, y = resid_robust), 
              color = I("red"), se = FALSE) + 
  geom_smooth(aes(x = yhat_nr, y = resid_norobust), 
              color = I("black"), se=FALSE) + 
  theme_bw()

NA
NA

coeftest(m_4, vcov = vcovHC(m_4, "HC1"))
coeftest(m_4, vcov = sandwich)
coeftest(m_4, vcov = vcovHC(m_4, "HC0"))


# check that "sandwich" returns HC0
coeftest(lmAPI, vcov = sandwich)                # robust; sandwich
coeftest(lmAPI, vcov = vcovHC(lmAPI, "HC0"))    # robust; HC0 

# check that the default robust var-cov matrix is HC3
coeftest(lmAPI, vcov = vcovHC(lmAPI))           # robust; HC3 
coeftest(lmAPI, vcov = vcovHC(lmAPI, "HC3"))    # robust; HC3 (default)

# reproduce the Stata default
coeftest(lmAPI, vcov = vcovHC(lmAPI, "HC1"))    # robust; HC1 (Stata default)

0.8 Descriptive Regression

As we discussed in the lecture, a powerful use of regression is to calculate an estimate of a variable accounting for multiple other factors simultaneously. Regression allows us to look at a key variable like a posttest score after accounting for key attributes of the test takers, like their demographic characteristics or their prior attendance.

0.8.1 Demographic Adjustments

One of the things people like to do is adjust for multiple student or school conditions simultaneously and then identify performance. We know all the parts necessary to do this - we define and fit a model, get its predictions, and then we plot the predictions using EDA techniques we have already mastered.

# Missing data results from dividing by 0, so we can set these at 0
full_data$ell_per[is.na(full_data$ell_per)] <- 0
full_data$swd_per[is.na(full_data$swd_per)] <- 0
# 
# Fit adjustment model
adj_model <- lm(ss2 ~ factor(grade) * factor(test_year) + subject + frl_per + 
                  ell_per + swd_per + n1 + locale_desc, data = full_data)
full_data$adj_ss2 <- predict(adj_model)

This code just selects some sample districts and creates two boxplots of their school/grade cohort scores before and after adjusting for demographics - as we looked at in the lecture.

example_districts <- c(155, 352, 277, 98, 323, 75, 44, 131, 280, 205)
p1 <- ggplot(full_data[full_data$distid %in% example_districts, ], 
       aes(y = adj_ss2, x = factor(distid))) + 
  geom_boxplot() + geom_jitter() + theme_bw() + 
  labs(title = "Adjusted Scores", x = "DISTID", y = "Adjusted SS2") + 
  ylim(1000, 1350)

p2 <- ggplot(full_data[full_data$distid %in% example_districts, ], 
       aes(y = ss2, x = factor(distid))) + 
  geom_boxplot() + geom_jitter() + theme_bw() + 
  labs(title = "Unadjusted Scores", x = "DISTID", y= "Observed SS2") + 
  ylim(1000, 1350) 


gridExtra::grid.arrange(p1, p2, ncol = 2)

We may also want to estimate the influence of one variable on another, but account for these conditional differences. One common question (h/t to Abigail) is the relationship between attendance and test scores. We often want to identify a critical threhold in this relationship. In the plot below, we use a scatterplot smoother between the posttest score and the school attendance rate to identify where the relationship between attendance and test scores changes. We expect there to be diminishing returns at both ends – additional days attending do not yield higher test scores at some point, the same way additional absences stop lowering test scores at some point.

But, we may be interested in understanding if this relationship is different after we have taken other factors about schools or students into account - so let’s compare our scores adjusted for school composition effects against the unadjusted scores, and look at how this relationship changes.


p1 <- ggplot(full_data[full_data$att_rate > 50,], 
       aes(y = adj_ss2, x = att_rate)) + 
    geom_jitter(alpha=1/7) + theme_bw() + 
    labs(title = "Adjusted Scores", x = "Pre-Test", y= "Adjusted SS2") + 
    geom_smooth(se=FALSE)  + ylim(1000, 1350) 

p2 <-  ggplot(full_data[full_data$att_rate > 50,], 
       aes(y = ss2, x = att_rate)) +  
  geom_jitter(alpha = 1/7) + theme_bw() + 
  labs(title = "Unadjusted Scores", x = "Attendance", y= "Observed SS2") + 
  geom_smooth(se=FALSE) + ylim(1000, 1350) 

gridExtra::grid.arrange(p1, p2, ncol = 2)

This plot does not display much difference, but it is seems clear that in the adjusted scores, the relationship with the posttest levels off at slightly higher attendance rates than the unadjusted scores. This suggests that after accounting for demographics, the cutpoint where attendance matters less is higher, and we may want to consider recommending a higher attendance threshold than we would if we only considered the unadjusted scores.

0.8.2 Simplified Value-Added

Value-added is another descriptive use of regression where we fit a model that we hope explains all the factors contributing to student growth that are observable, and then we look at the residual error from that model, the unobserved error, and interpret it as attributable to the school or teacher etc.


vam_model <- lm(ss2 ~ ss1 + factor(grade) * factor(test_year) + frl_per + 
                  ell_per + swd_per + n1 + locale_desc + att_rate + subject +
                  factor(distid), data = full_data)
full_data$vam_ss2 <- predict(vam_model)
full_data$vam_score <- rstandard(vam_model)


ggplot(full_data[full_data$distid %in% example_districts, ], 
       aes(y = vam_score, x = factor(distid))) + 
  geom_boxplot(color="gray80") + geom_jitter() + theme_bw() + 
  labs(title = "Vam Scores", x = "DISTID", y= "Observed SS2")

NA
NA

VAM models are often more complex and include post-estimation adjustments of the residuals to account for the sample size or aggregation of schools into districts or classrooms into schools. But, this is the basic format of value-added modeling and the intuition that underlies it.

As a final question - how much better does this value-added model fit the data than other models we have used in the guide? How might you evaluate that? What are the consequences of that for interpretation?

0.9 BONUS: Testing Many Models

Above, we have shown regression diagnostics for just one model. But, now our colleague has proposed fifty. The model above shows signs of misspecification and heteroskedasticity - which gives us reason to consider testing the other 49 models and comparing them.

To do this, we will use an R function that applies the test to each model and returns a data.frame with one row per model and one column per test. It’s OK if you don’t feel comfortable writing and editing functions just yet - but this is a useful example of how the flexibility to writing functions can allow you to do scale your analysis from one to many models easily.

First the function, we’ll call it regress_battery - as in a battery of regression tests.

# Functions are just named objects in R of a special type
# Here we name our function, and we say that it is a function with two "arguments"
# the first model, the second order_var. The default value of order_var is NULL
regress_battery <- function(model, order_var = NULL) {
  # model = a regression model object, of class lm
  # order_var = a vector of values to reorder the regression test by
  
  # First, extract the residuals from the model, we need these for some 
  # types of regression tests
    ordered_resids <- residuals(model)
  if (!is.null(order_var)) { # if order is given, reorder the residuals
      ordered_resids <- ordered_resids[order(order_var)]
      order_by <- order_var[1]
  } else {
    # If no order is given, specify that the order value is NA
      order_by <- NA
  }
  suppressWarnings({ # avoid some annoying output to the console caused by the tests
    out <- data.frame(
            order_by = order_by, # give an example value of the sort by data
            breusch_pagan = bptest(model)$p.value, # p-value from bptest
            goldfeld_quandt = gqtest(model)$p.value,
            harvey_collier = harvtest(model, order.by = order_var)$p.value,
            reset = reset(model, power = 2:4)$p.value,
            rainbow = raintest(model, order.by = order_var)$p.value,
            box_test = Box.test(ordered_resids)$p.value,
            ljung_test = Box.test(ordered_resids, type = "Ljung")$p.value,
            durbin_watson = dwtest(model, order.by = order_var)$p.value,
            breusch_godfrey = bgtest(model, order.by = order_var)$p.value
          )
    })
  
  out[, 2:10] <- apply(out[, 2:10], 2, round, 3) # round p-values to the 0.001
  row.names(out) <- NULL
  return(out) # return the data.frame
}

# Now call the function
regress_battery(m1, order_var = by_group$data[[15]]$distid)

This example model shows signs of heteroskedacticity and perhaps a need for polynomial terms, but otherwise checks out. However, it isn’t about how a single model performs - we want to assess collectively how the fifty models on our fifty subsamples perform.

To test all of them, we use the rowwise() operator to operate on each row of our by_group data frame. We take one model and the data set it was fit to from the model and data columns respectively, and then produce a new data.frame with 50 rows of regression diagnostiics.

# Group the data and appply the function to each model
mod_test_ss1 <- by_group %>%  
  rowwise() %>% 
  do(regress_battery(model = .$base_model, order_var = .$data$ss1))
mod_test_ss1$order_by <- "ss1"

mod_test_ss1 <- bind_cols(by_group %>% 
                            dplyr::select(grade, subject, test_year), 
                          mod_test_ss1)

# Group the data and appply the function to each model
mod_test_distid <- by_group %>%  
  rowwise() %>% 
  do(regress_battery(model = .$base_model, order_var = .$data$distid))
mod_test_distid$order_by <- "distid"

mod_test_distid <- bind_cols(by_group %>% dplyr::select(grade, subject, test_year), 
                          mod_test_distid)

If you want, you can inspect the results for these models.

Now we reshape and plot

plotdf <- bind_rows(mod_test_ss1, mod_test_distid) %>% 
  as.data.frame

names(plotdf)[5:13] <- paste0("p_value", ".", names(plotdf)[5:13])

plotdf <- reshape(plotdf, direction = "long",  
                  varying = 5:13, 
                  sep = ".", new.row.names = NULL)

custom_brks <- c(0, 0.1, 0.2, 0.5, 0.8, 1)

ggplot(plotdf, aes(x = p_value)) + geom_histogram(breaks = custom_brks) + 
  facet_grid(order_by~time) + theme_bw()

This chart shows us for each test how many models out of fifty are at which p-value. The “Breusch-Pagan” column shows that almost all models are grouped at a low p-value (e.g. indicating the diagnostic is finding a problem). On the other hand, the rainbow test shows that almost no models when ordered by distid and almost all models when ordered by ss1 have low p-values indicating the diagnostic has found a problem.

The other tests are much more mixed - a set of models fail and a set of models pass quite a bit. The results are not conclusive - across the fifty models the diagnostics show that some models fail some tests and pass others.

0.10 BONUS: Over the Top Outlier Plot

To make a fun plot: Of interest to us is the is.inf object, which for each row in our data provides six true/false values indicating whether an observation is influential on one of 5 measures plus an additional measure for every coefficient in our model.


# Define outlier status across these variables
sch_score_m1$outlier <- sch_score_m1$.hat + sch_score_m1$.cooksd + sch_score_m1$.std.resid

ggplot(sch_score_m1) + aes(x = ss1, y = ss2, color = outlier^2, 
                        size = outlier^2) + 
  geom_point(alpha = I(0.33)) +
  scale_colour_gradient(low = "blue", high="red", breaks = c(0, 1, 4, 9))+
  scale_radius(range = c(0,9), breaks = c(0, 1, 4, 9)) +
  guides(color = guide_legend(), size = guide_legend()) +
  labs(color = "Strength of Infl.", size = "Strength of Infl.", 
       title = "Influence of Observations on Regression Line") +
  geom_smooth(se=FALSE, show.legend = FALSE) +
  geom_smooth(method= "lm", se=FALSE, show.legend = FALSE, 
              color = I("black"), linetype = 2) +
  geom_hline(yintercept = mean(sch_score_m1$ss2)) + 
  geom_vline(xintercept = mean(sch_score_m1$ss1)) + theme_bw()

LS0tDQp0aXRsZTogIlIgTm90ZWJvb2siDQpvdXRwdXQ6DQogIGh0bWxfZG9jdW1lbnQ6DQogICAgZGZfcHJpbnQ6IHBhZ2VkDQogIGh0bWxfbm90ZWJvb2s6DQogICAgbnVtYmVyX3NlY3Rpb25zOiB5ZXMNCmVkaXRvcl9vcHRpb25zOg0KICBjaHVua19vdXRwdXRfdHlwZTogaW5saW5lDQotLS0NCg0KDQojIyBJbnRyb2R1Y3Rpb24NCg0KRm9yIG91ciBzZWNvbmQgZ3VpZGVkIGFuYWx5c2lzLCB3ZSB3aWxsIGJlIHdvcmtpbmcgb24gYWR2YW5jaW5nIHRoZSBzZWFyY2ggZm9yIA0KYSBtb2RlbCB0byBpZGVudGlmeSBhZ2dyZWdhdGUgdGVzdCBzY29yZSBhbm9tYWxpZXMgaW4gb3VyIHN0YXRlLiBJbiBvdXIgcHJldmlvdXMgDQptb2R1bGUgd2UgZGV0ZXJtaW5lZCB0aGF0IGEgb25lLXNpemUtZml0cy1hbGwgYXBwcm9hY2ggbGVkIHRvIGEgbW9kZWwgdGhhdCANCm1vc3RseSBpZGVudGlmaWVkIHNjaG9vbHMgaW4gbG93ZXIgZ3JhZGVzLCBhbmQgbW9zdCBvZnRlbiBpZGVudGlmaWVkIHNjaG9vbHMgDQp3aXRoIHNtYWxsIGNsYXNzIHNpemVzLiBFdmVuIHdpdGggdGhlIGFkZGl0aW9uIG9mIGV4dHJhIGRhdGEsIHRoZSBkYXRhIHRlYW0gDQpoYXMgbm90IHlldCBmb3VuZCBhIG1vZGVsIHRoYXQgaWRlbnRpZmllcyBhYm5vcm1hbGl0aWVzIHRoYXQgYXJlIG5vdCBjb3JyZWxhdGVkIA0Kd2l0aCBzY2hvb2wgc2l6ZSBhbmQgc2Nob29sIGdyYWRlIGxldmVsLiANCg0KWW91IGhhdmUgYmVlbiBnaXZlbiB0aGUgdGltZSB0byBleHBsb3JlIHNvbWUgYWx0ZXJuYXRpdmUgYXBwcm9hY2hlcyB0byB0aGlzIA0Kd29yayAtIGluY2x1ZGluZyBzdGVwcGluZyBhd2F5IGZyb20gdGhlIG9uZS1zaXplLWZpdHMtYWxsIGFwcHJvYWNoIGFuZCBtb3ZpbmcgDQp0b3dhcmQgYSBtdWx0aS1tb2RlbCBhcHByb2FjaC4gWW91ciBzdXBlcnZpc29yIGhhcyB0b2xkIHlvdSB0aGF0IGJlZm9yZSBjb25zaWRlcmluZyANCnRoZSB0YXNrIG5vdCBwb3NzaWJsZSB3aXRoIHRoZSBkYXRhIGF0IGhhbmQsIHNoZSdkIGxpa2UgeW91IHRvIGxvb2sgYXQgdGhlIA0KcG9zc2liaWxpdHkgb2YgZml0dGluZyBzZXBhcmF0ZSBtb2RlbHMgdG8gc2VwYXJhdGUgc2FtcGxlcyBvZiB0aGUgZGF0YSBhbmQgDQpzZWVpbmcgd2hhdCB0aGlzIGNoYW5nZXMuIA0KDQojIyMgU2V0dXAgYW5kIFJlYWQgRGF0YQ0KDQpGb3IgdGhpcyB0YXNrLCB0aGUgZGF0YSB0ZWFtIG5vdyBoYXMgYSBmdWxsIGRhdGFzZXQgYXZhaWxhYmxlIHRvIGl0IC0gSVQgaGFzIA0KZmlsbGVkIGVhY2ggb2YgdGhlIHJlcXVlc3RzIGZvciBhZGRpdGlvbmFsIGRhdGEuIE91ciBmaXJzdCB0YXNrIGlzIHRvIGpvaW4gDQp0aGVzZSBkYXRhc2V0cyBhbmQgY3JlYXRlIGFuIGFuYWx5c2lzIGRhdGFzZXQuIA0KDQoNCmBgYHtyIHNldHVwLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KbGlicmFyeShkcGx5cikNCiMgUmVhZCBpbiB0aGUgZGF0YQ0KIyBzdHVfZGF0YSA8LSByZWFkLmNzdigiZGF0YS9zaW1fYXNzZXNzbWVudF9kYXRhLmNzdiIsIHN0cmluZ3NBc0ZhY3RvcnMgPSBGQUxTRSkNCmxvYWQoImRhdGEvY29ob3J0X3Rlc3Rfc2NvcmVzLnJkYSIpDQpoZWFkKHNjaF9zY29yZSwgYWRkcm93bnVtcyA9IEZBTFNFKQ0KYGBgDQoNCg0KYGBge3IgbG9hZGRhdGF9DQpsb2FkKCJkYXRhL2FnZW5jeS5yZGEiKQ0KbG9hZCgiZGF0YS9hdHRlbmQucmRhIikNCmxvYWQoImRhdGEvZXhwZWwucmRhIikNCmxvYWQoImRhdGEvZW5yb2xsbWVudC5yZGEiKQ0KbG9hZCgiZGF0YS9zdGFmZi5yZGEiKQ0KbHMoKQ0KYGBgDQoNCmBgYHtyIGpvaW5kYXRhfQ0KZnVsbF9kYXRhIDwtIGlubmVyX2pvaW4oc2NoX3Njb3JlLCBhZ2VuY3ksIA0KICAgICAgICAgICAgICAgICAgICAgICAgYnkgPSBjKCJkaXN0aWQiLCAic2NoaWQiLCAidGVzdF95ZWFyIikpDQpmdWxsX2RhdGEgPC0gaW5uZXJfam9pbihmdWxsX2RhdGEsIGF0dGVuZCwgDQogICAgICAgICAgICAgICAgICAgICAgICBieSA9IGMoImRpc3RpZCIsICJzY2hpZCIsICJ0ZXN0X3llYXIiLCAic2Nob29sX3llYXIiKSkNCmZ1bGxfZGF0YSA8LSBpbm5lcl9qb2luKGZ1bGxfZGF0YSwgZXhwZWwsIA0KICAgICAgICAgICAgICAgICAgICAgICAgYnkgPSBjKCJkaXN0aWQiLCAic2NoaWQiLCAidGVzdF95ZWFyIiwgInNjaG9vbF95ZWFyIikpDQpmdWxsX2RhdGEgPC0gaW5uZXJfam9pbihmdWxsX2RhdGEgJT4lIGRwbHlyOjpzZWxlY3QoLWVucm9sbG1lbnQpLCBmdWxsX2Vucm9sbCwgDQogICAgICAgICAgICAgICAgICAgICAgICBieSA9IGMoImRpc3RpZCIsICJzY2hpZCIsICJ0ZXN0X3llYXIiLCAic2Nob29sX3llYXIiKSkNCmZ1bGxfZGF0YSA8LSBpbm5lcl9qb2luKGZ1bGxfZGF0YSwgc3RhZmYsIA0KICAgICAgICAgICAgICAgICAgICAgICAgYnkgPSBjKCJkaXN0aWQiLCAic2NoaWQiLCAidGVzdF95ZWFyIiwgInNjaG9vbF95ZWFyIikpDQpzdHIoZnVsbF9kYXRhKQ0KYGBgDQoNCmBgYHtyIGNsZWFudXB3b3Jrc3BhY2V9DQpybShhZ2VuY3ksIGF0dGVuZCwgc2NoX3Njb3JlLCBleHBlbCwgc3RhZmYsIGZ1bGxfZW5yb2xsKQ0KYGBgDQoNCg0KRGVzcGl0ZSB0aGUgYWRkaXRpb25hbCBkYXRhIC0gdGhlcmUgaXMgcHJlc3N1cmUgdG8ga2VlcCB0aGUgbW9kZWwgc2ltcGxlLiBZb3VyIA0KdGVhbSBmZWVscyBpdCB3aWxsIGJlIGVhc2llciB0byBleHBsYWluIHRvIGV4dGVybmFsIHN0YWtlaG9sZGVycyBhbmQgd2lsbCBjcmVhdGUgDQpmZXdlciBvcHBvcnR1bml0aWVzIGZvciBzY2hvb2xzIHRvIGRpc3B1dGUgdGhlaXIgY2xhaW1zLiBUaGUgY29sbGVhZ3VlIHdobyANCnByb3Bvc2VkIHRoZSBmaXJzdCBvbmUtc2l6ZS1maXRzLWFsbCBoYXMgcHJvcG9zZWQgYSBzaW1wbGUgYWx0ZXJuYXRpdmUgaW4gcmVzcG9uc2UgDQp0byB5b3VyIGZlZWRiYWNrIGFuZCBzdWJzZXF1ZW50IGFuYWx5c2VzLiANCg0KVGhlIG5ldyBtb2RlbCBpcyB0byBzaW1wbHkgZml0IGEgc2VwYXJhdGUgcmVncmVzc2lvbiBvZiB0aGUgbGFnZ2VkIHNjYWxlIHNjb3JlIA0Kb24gdGhlIGN1cnJlbnQgc2NhbGUgc2NvcmUgZm9yIGVhY2ggc2VwYXJhdGUgZ3JhZGUtc3ViamVjdC15ZWFyIGNvbWJpbmF0aW9uLiANCllvdXIgY29sbGVhZ3VlIGhhcyBnaXZlbiB5b3UgdGhlIGNvZGUgYmVsb3cgdG8gaXRlcmF0ZSB0aHJvdWdoIHlvdXIgZGF0YSBhbmQgDQpmaXQgdGhlc2UgY29tYmluYXRpb25zLiANCg0KRmlyc3QsIGxldCdzIHNlZSBob3cgbWFueSBtb2RlbHMgdGhpcyBnaXZlcyB1czogDQoNCmBgYHtyIGNvdW50cm93c30NCmZ1bGxfZGF0YSAlPiUgZHBseXI6OnNlbGVjdCh0ZXN0X3llYXIsIGdyYWRlLCBzdWJqZWN0KSAlPiUgDQogIGRpc3RpbmN0ICU+JSBucm93DQpgYGANCg0KQW5kLCBob3cgbWFueSByb3dzIHBlciBtb2RlbDoNCg0KYGBge3IgY291bnRyb3dzcGVyZ3JvdXB9DQpmdWxsX2RhdGEgJT4lIGRwbHlyOjpzZWxlY3QodGVzdF95ZWFyLCBncmFkZSwgc3ViamVjdCkgJT4lIA0KICBncm91cF9ieSh0ZXN0X3llYXIsIGdyYWRlLCBzdWJqZWN0KSAlPiUNCiAgc3VtbWFyaXplKGNvdW50ID0gbigpKSAlPiUgDQogIHB1bGwoY291bnQpICU+JSBzdW1tYXJ5DQpgYGANCg0KV2UgaGF2ZSBhcyBmZXcgYXMgMTUyIG9ic2VydmF0aW9ucyBpbiBzb21lIGdyb3VwcyBhbmQgYXMgbWFueSBhcyA0NjkuIFNpbmNlIHdlIA0Ka25vdyByZWdyZXNzaW9uIG1vZGVscyBhcmUgZWZmaWNpZW50IGFuZCBkbyBub3QgaGF2ZSBsYXJnZSBkYXRhIHJlcXVpcmVtZW50cywgDQp0aGlzIGlzIG5vdCBhIGNvbmNlcm4uIA0KDQpOb3csIGxldCdzIGxvb2sgYXQgYSBsb29wIHdlIGNhbiB1c2UgdG8gZml0IHRoZSBzYW1lIG1vZGVsIHRvIGVhY2ggb2Ygb3VyIDUwIA0Kc3Vic2V0cyBvZiBkYXRhIGVmZmljaWVudGx5Og0KDQoNCmBgYHtyIGZpZnR5bW9kZWxsb29wfQ0KIyBMb2FkIHRoZSBwYWNrYWdlcyB3ZSBuZWVkIHRvIGZpdCBtdWx0aXBsZSBtb2RlbHMNCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShicm9vbSkgIyB0byBtYW5pcHVsYXRlIHJlZ3Jlc3Npb24gbW9kZWxzDQpsaWJyYXJ5KG1vZGVscikgIyB0byBmaXQgbXVsdGlwbGUgbW9kZWxzIGFuZCBnZXQgdGhlaXIgYXR0cmlidXRlcyBlYXNpbHkNCg0KIyBEZWZpbmUgYSBncm91cGVkIGRhdGFmcmFtZSB1c2luZyBvbmx5IHRoZSBjb2x1bW5zIHdlIG5lZWQNCiMgZ3JvdXAgYnkgdGhlIGdyYWRlLCBzdWJqZWN0LCBhbmQgeWVhcg0KIyBUaGVuIG5lc3Qgc28gdGhlIGRhdGEgaGFuZ3MgdG9nZXRoZXIgaW4gb25lIGxpc3QNCmJ5X2dyb3VwIDwtIGZ1bGxfZGF0YVssIDE6MTBdICU+JQ0KICBncm91cF9ieShncmFkZSwgc3ViamVjdCwgdGVzdF95ZWFyKSAlPiUNCiAgbmVzdCgpDQoNCiMgRGVmaW5lIGEgc2ltcGxlIGZ1bmN0aW9uIHRoYXQgZml0cyBvdXIgbW9kZWwNCnNpbXBsZV9tb2RlbCA8LSBmdW5jdGlvbihkZikgew0KICBsbShzczIgfiBzczEsIGRhdGEgPSBkZikNCn0NCg0KIyBBcHBseSB0aGF0IG1vZGVsIHRvIGVhY2ggZ3JvdXAgaW4gb3VyIGRhdGFzZXQgdXNpbmcgdGhlIGBtYXBgIGZ1bmN0aW9uDQojIFRoaXMgaXMgZXF1aXZhbGVudCB0byBhIGxvb3AsIGJ1dCBtb3JlIGVmZmljaWVudCB0byB3cml0ZQ0KYnlfZ3JvdXAgPC0gYnlfZ3JvdXAgJT4lDQogIG11dGF0ZShtb2RlbCA9IG1hcChkYXRhLCBzaW1wbGVfbW9kZWwpKQ0KDQojIFRvIHVuZGVyc3RhbmQgb3VyIG1vZGVsKHMpIC0gdXNlIHRoZSBgZ2xhbmNlYCBmdW5jdGlvbiB0byBjb21wdXRlIHNvbWUgDQojIHN1bW1hcnkgc3RhdGlzdGljcyBmb3IgZWFjaCBtb2RlbA0KZ2xhbmNlIDwtIGJ5X2dyb3VwICU+JQ0KICBtdXRhdGUoZ2xhbmNlID0gbWFwKG1vZGVsLCBicm9vbTo6Z2xhbmNlKSkgJT4lDQogIHVubmVzdChnbGFuY2UsIC5kcm9wID0gVFJVRSkNCg0KIyBEaXNwbGF5IHRoZSBvdXRwdXQNCmFycmFuZ2UoZ2xhbmNlLCBkZXNjKHIuc3F1YXJlZCkpDQpgYGANCg0KDQpUaGVyZSBpcyBhIGxvdCB0byB1bnBhY2sgaW4gdGhpcyBiaXQgb2YgY29kZS4gV2UncmUgZ29pbmcgdG8gZXhwbG9yZSBpdCBpbiBkZXRhaWwgDQphbmQgdGhlbiBtb2RpZnkgZGlmZmVyZW50IHBpZWNlcyBvZiBpdCB0byBkbyBvdXIgYW5hbHlzaXMuIFRoaXMgaXMgd2hlcmUgdGhlIA0KaXRlcmF0aXZlIHBvd2VyIG9mIHN0YXRpc3RpY2FsIGNvbXB1dGluZyByZWFsbHkgc2hpbmVzLCBiZWNhdXNlIHdlIGNhbiBkbyBtYW55IA0Kb3BlcmF0aW9ucyByZXBlYXRlZGx5IGFjcm9zcyBvdXIgc3ViLXNhbXBsZXMsIGNvbGxlY3QgdGhlIHJlc3VsdHMsIGFuZCBjb21wYXJlIA0KYW5kIGNvbXB1dGUgdGhlIHJlc3VsdHMgb2Ygb3VyIGFuYWx5c2VzLiANCg0KVGhlIGZpcnN0IHRoaW5nIHdlIHdhbnQgdG8gZG8gaXMgZ2F0aGVyIG91ciBkYXRhIGludG8gc3Vic2FtcGxlcyBncm91cGVkIGJ5IA0KdGhlIGdyYWRlLCBzdWJqZWN0LCBhbmQgeWVhciBvZiB0aGUgdGVzdC4gVGhpcyBjb2RlIHRha2VzIG91ciBmdWxsIGRhdGEgc2V0LCANCmdyb3VwcyBpdCBieSB0aGVzZSB2YXJpYWJsZXMsIGFuZCB0aGVuIG5lc3RzIGVhY2ggb2JzZXJ2YXRpb24gZm9yIGVhY2ggZ3JvdXAgDQppbnRvIGEgbmV3IGRhdGFmcmFtZSBhbmQgc3RvcmVzIHRoYXQgZGF0YWZyYW1lIGFzIGEgY29sdW1uLiBUaGUgcmVzdWx0aW5nIGRhdGEgDQpzdHJ1Y3R1cmUgaXMgYSBkYXRhZnJhbWUgd2l0aCA1MCByb3dzLCBidXQgb25lIG9mIHRoZSBjb2x1bW5zIGNvbnRhaW5zIGEgDQpkYXRhc2V0IHdpdGhpbiBpdCB0aGF0IGNvcnJlc3BvbmRzIHRvIHRoZSBmdWxsIG9ic2VydmF0aW9ucyBmb3IgdGhhdCBjb21iaW5hdGlvbiANCm9mIGdyYWRlLCBzdWJqZWN0LCBhbmQgdGVzdCB5ZWFyLiANCg0KSXQncyBhIG5lYXQgdHJpY2sgdGhhdCBpcyB1c2VmdWwgaW4gZWR1Y2F0aW9uIGRhdGEgYW5hbHlzaXMgd2hlcmUgc28gbWFueSBvZiBvdXIgDQpxdWVzdGlvbnMgYXJlIGFib3V0IG5lc3RlZCBzdHJ1Y3R1cmVzIG9mIG9ic2VydmF0aW9ucyAtIHN0dWRlbnRzIGluIGNsYXNzcm9vbXMsIA0KY2xhc3Nyb29tcyBpbiBzY2hvb2xzLCBzY2hvb2xzIGluIGRpc3RyaWN0cywgZXRjLiANCg0KDQpgYGB7ciBjcmVhdGVncm91cGVkZGZ9DQpieV9ncm91cCA8LSBmdWxsX2RhdGFbLCAxOjEwXSAlPiUNCiAgZ3JvdXBfYnkoZ3JhZGUsIHN1YmplY3QsIHRlc3RfeWVhcikgJT4lDQogIG5lc3QoKQ0KDQpoZWFkKGJ5X2dyb3VwKQ0KYGBgDQoNClRoZSBhZHZhbnRhZ2UgaGVyZSBpcyB0aGF0IHdlIGFscmVhZHkga25vdyBob3cgdG8gdGVsbCBSIHRvIGRvIHRoaW5ncyB0byBhbiANCmVudGlyZSBjb2x1bW4gb2YgZGF0YSBhdCBvbmNlLCBzbyB3ZSBjYW4gdXNlIHRob3NlIHNhbWUgc2tpbGxzIHRvIHJhcGlkbHkgDQptYWtlIGNhbGN1bGF0aW9ucyBvbiBlYWNoIG9mIHRoZXNlIGZpZnR5IHNlcGFyYXRlIGRhdGFzZXRzIGFsbCB3aXRob3V0IGhhdmluZyANCnRvIHdyaXRlIGFuIGV4cGxpY2l0IGxvb3AuIA0KDQpUaGUgbmV4dCBzZXQgb2YgY29kZSBkZWZpbmVzIG91ciBtb2RlbCBhcyBhIGZ1bmN0aW9uIGNhbGxlZCwgdW5jcmVhdGl2ZWx5LCANCmBzaW1wbGVfbW9kZWxgLiBGdW5jdGlvbiBkZWZpbml0aW9ucyBhcmUgZWFzeSBpbiBSLCBvbmNlIHlvdSByZWFkIGluIHRoZSANCmZ1bmN0aW9uIGRlZmluaXRpb24sIHlvdSB3aWxsIHNlZSBpdCBhcHBlYXIgaW4gdGhlICJGdW5jdGlvbnMiIHNlY3Rpb24gb2YgeW91ciANClJTdHVkaW8gZW52aXJvbm1lbnQgcGFuZSwgYW5kIHlvdSBjYW4gdXNlIGl0IGp1c3QgbGlrZSBhbnkgb3RoZXIgY29tbWFuZCBpbiBSLg0KDQpXcml0aW5nIGZ1bmN0aW9ucyBjYW4gYmUgYSBiaXQgdHJpY2t5IGF0IGZpcnN0LiBUaGlzIGZ1bmN0aW9uIGlzIG5vdCBpZGVhbCBiZWNhdXNlIA0KaXQgaGFzIHZhcmlhYmxlcyB3aXRoaW4gaXQgKGBzczJgIGFuZCBgc3MxYCkgdGhhdCBhcmUgbm90IGRlZmluZWQgaW4gdGhlIGZ1bmN0aW9uIA0KaXRzZWxmIC0tIGl0IGFzc3VtZXMgdGhleSB3aWxsIGV4aXN0LiBJdCBpcyBzYWZlIGZvciB1cyB0byB1c2UgaGVyZSBiZWNhdXNlIA0Kd2Uga25vdyB0aG9zZSB2YXJpYWJsZXMgd2lsbCBhbHdheXMgZXhpc3QuIA0KDQpUaGUgbmV4dCBsaW5lIG9mIGNvZGUgc2ltcGx5IHRlbGxzIFIgdG8gY3JlYXRlIGEgbmV3IGNvbHVtbiBpbiBvdXIgZGF0YXNldCANCmNhbGxlZCBgbW9kZWxgLCBhbmQgdG8gY3JlYXRlIHRoYXQgZGF0YXNldCBpdCBzaG91bGQgYXBwbHkgdGhlIGZ1bmN0aW9uIHNpbXBsZV9tb2RlbCANCnRvIGVhY2ggZWxlbWVudCBvZiB0aGUgYGRhdGFgIGNvbHVtbiBhbmQgc3RvcmUgdGhlIG91dHB1dCAoYSBsaW5lYXIgbW9kZWwpIGluIA0KdGhlIG5ldyBjb2x1bW4uIFRoZSBgbWFwYCBmdW5jdGlvbiBpcyBmcm9tIHRoZSBgcHVycnJgIHBhY2thZ2UgYW5kIHRlbGxzIFIgdG8gDQp1c2UgdGhlIGBzaW1wbGVfbW9kZWxgIGZ1bmN0aW9uIG9uIGVhY2ggZWxlbWVudCBvZiB0aGUgYGRhdGFgIGNvbHVtbi4gVGhlIHRlcm0gDQpgbWFwYCByZWZlcnMgdG8gdGhlIGZhY3QgdGhhdCB3ZSBhcmUgIm1hcHBpbmciIHRoZSBmdW5jdGlvbiBgc2ltcGxlX21vZGVsYCB0byANCmVhY2ggZWxlbWVudCBvZiB0aGUgYGRhdGFgIGNvbHVtbi4gDQoNCmBgYHtyIGZpdG1vZGVsdG9lYWNoZ3JvdXB9DQpzaW1wbGVfbW9kZWwgPC0gZnVuY3Rpb24oZGYpIHsNCiAgIyBkZiBpcyBhIHVzZXIgc3BlY2lmaWVkIHBhcmFtZXRlciB0byB0aGUgZnVuY3Rpb24NCiAgIyBzczIgYW5kIHNzMSBhcmUgdmFyaWFibGVzIHRoYXQgbXVzdCBiZSBwcmVzZW50IGluIHRoZSBkZiBwcm92aWRlZCBieSB0aGUgdXNlcg0KICBsbShzczIgfiBzczEsIGRhdGEgPSBkZikNCn0NCg0KaGVhZChieV9ncm91cCkNCg0KYnlfZ3JvdXAgPC0gYnlfZ3JvdXAgJT4lDQogICMgdGFrZSBlYWNoIGVsZW1lbnQgb2YgdGhlIGRhdGEgY29sdW1uLCBhbmQgcGFzcyBpdCBpbmRpdmlkdWFsbHkgYXMgdGhlIA0KICAjIGZpcnN0IGFyZ3VtZW50IHRvIHRoZSBzaW1wbGVfbW9kZWwgZnVuY3Rpb24NCiAgbXV0YXRlKG1vZGVsID0gbWFwKGRhdGEsIHNpbXBsZV9tb2RlbCkpDQoNCmhlYWQoYnlfZ3JvdXApDQoNCmBgYA0KDQpTbyBub3cgd2UgaGF2ZSBjcmVhdGVkIGZvciBlYWNoIGdyb3VwIHR3byBuZXcgY29sdW1ucywgYnV0IGluc3RlYWQgb2Ygc3RvcmluZyANCnZhbHVlcywgdGhleSBzdG9yZSBhIGRhdGFzZXQgYW5kIGEgc3RhdGlzdGljYWwgbW9kZWwgcmVzcGVjdGl2ZWx5LiBHbyBhaGVhZCANCmFuZCBsb29rIGZvciB5b3Vyc2VsZjoNCg0KYGBge3IgY29uZmlybW1vZGVsaW5kYXRhZnJhbWV9DQpzdW1tYXJ5KGJ5X2dyb3VwJGRhdGFbWzEwXV0pICMgMTAgYW5kIDExIGFyZSBqdXN0IGluZGV4ZXMgdGhhdCBzYXkgd2hpY2ggZQ0KIyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVsZW1lbnQgd2Ugd2FudCwgaGVyZSB0aGUgMTB0aCBlbGVtZW50DQpzdW1tYXJ5KGJ5X2dyb3VwJGRhdGFbWzExXV0pICMgaGVyZSB0aGUgMTF0aCBlbGVtZW50DQoNCnN1bW1hcnkoYnlfZ3JvdXAkbW9kZWxbWzFdXSkgIyBoZXJlIHRoZSBmaXJzdCBlbGVtZW50DQpzdW1tYXJ5KGJ5X2dyb3VwJG1vZGVsW1syXV0pDQpgYGANCg0KUHJldHR5IG5lYXQhIEJ1dCwgaXQncyBub3QgZW5vdWdoIHRvIGNhbGN1bGF0ZSA1MCByZWdyZXNzaW9uIG1vZGVscywgd2Ugd2FudCB0byANCnRlc3QsIGNvbXBhcmUsIGFuZCBwZXJoYXBzIG1vZGlmeSB0aGVtLiBXaXRoIHRoZSB3YXkgb3VyIGRhdGEgYXJlIA0Kbm93IHN0cnVjdHVyZWQsIHdlIGNhbiBlYXNpbHkgZXh0cmFjdCB0aGUgZmVhdHVyZXMgd2UgYXJlIGludGVyZXN0ZWQgDQppbiBmcm9tIGVhY2ggb2YgdGhlIG1vZGVscyBhbmQgc3RvcmUgdGhlbSBmb3IgZnVydGhlciBhbmFseXNpcy4gTGV0J3Mgc2F5IHdlIA0Kd2FudGVkIHRvIGxvb2sgYXQgaG93IHRoZSBpbnRlcmNlcHQgYW5kIHNsb3BlIHRlcm1zIHZhcnkgYWNyb3NzIHRoZSBmaWZ0eSBncm91cHMsIA0Kd2UgY2FuIGRvIHRoYXQhDQoNCkFnYWluIHdlIHVzZSB0aGUgYG1hcGAgZnVuY3Rpb24gdG8gZXh0cmFjdCB0aGUgY29lZmZpY2llbnRzIGZyb20gb3VyIG1vZGVsIC0gaW4gDQp0aGlzIGNhc2UsIHdlIHdpbGwgaGF2ZSB0d28gY29lZmZpY2llbnRzLiBXZSB1c2UgdGhlIFIgZnVuY3Rpb24gYGNvZWZgIGFuZCB3ZSBtYXAgDQppdCB0byBlYWNoIGVsZW1lbnQgb2YgbW9kZWwuIA0KDQpgYGB7ciBnZXRlc3RpbWF0ZXNmcm9tbW9kZWxzfQ0KDQpieV9ncm91cCA8LSBpbm5lcl9qb2luKA0KICBieV9ncm91cCwgIyBvdXIgb3JpZ2luYWwgZGF0YQ0KICAgYnlfZ3JvdXAgJT4lDQogICAgICBtdXRhdGUoY29lZnMgPSBtYXAobW9kZWwsIGNvZWYpKSAlPiUgICMgZ2V0IHRoZSBjb2VmZmljaWVudHMgb3V0IG9mIHRoZSBtb2RlbA0KICAgICAgZ3JvdXBfYnkoZ3JhZGUsIHN1YmplY3QsIHRlc3RfeWVhcikgJT4lICMgZ3JvdXAgdGhlIGRhdGEgDQogICAgICBkbyhkYXRhLmZyYW1lKHQodW5saXN0KC4kY29lZnMpKSkpICMgc3BsaXQgZWFjaCBjb2VmZmljaWVudCBpbnRvIGEgbmV3IGNvbHVtbg0KKQ0KDQpuYW1lcyhieV9ncm91cCkNCiMgUmVwbGFjZSB0aGUgNnRoIGVsZW1lbnRzIG5hbWUNCm5hbWVzKGJ5X2dyb3VwKVs2XSA8LSAiZXN0X2ludGVyY2VwdCINCiMgUmVwbGFjZSB0aGUgN3RoIGVsZW1lbnRzIG5hbWUNCm5hbWVzKGJ5X2dyb3VwKVs3XSA8LSAiZXN0X3NzMSINCg0KYGBgDQoNCkFuZCBub3cgd2UgY2FuIHBsb3QgdGhlIGRpZmZlcmVuY2VzIGFuZCBkbyBzb21lIGdyYXBoaWNhbCBFREEuIA0KDQpgYGB7ciBwbG90ZmlmdHltb2RlbGVzdGltYXRlc30NCmdncGxvdChieV9ncm91cCwgYWVzKHggPSBlc3RfaW50ZXJjZXB0KSkgKyBnZW9tX2RlbnNpdHkoKSArIHRoZW1lX2J3KCkNCmdncGxvdChieV9ncm91cCwgYWVzKHggPSBlc3Rfc3MxKSkgKyBnZW9tX2RlbnNpdHkoKSArIHRoZW1lX2J3KCkNCg0KZ2dwbG90KGJ5X2dyb3VwKSArIHNjYWxlX3hfY29udGludW91cyhsaW1pdHMgPSBjKDcwMCwgMTYwMCkpICsNCiAgc2NhbGVfeV9jb250aW51b3VzKGxpbWl0cyA9IGMoNzAwLCAxNjAwKSkgKyANCiAgZ2VvbV9hYmxpbmUoZGF0YSA9IGJ5X2dyb3VwLCANCiAgICAgICAgICAgICAgYWVzKHNsb3BlID0gZXN0X3NzMSwgaW50ZXJjZXB0ID0gZXN0X2ludGVyY2VwdCksIA0KICAgICAgICAgICAgICBhbHBoYSA9IEkoMC41KSkgKyB0aGVtZV9idygpDQpgYGANCg0KT0ssIHNvIHdlJ3ZlIGdvdCBvdXIgYmFzaWMgc3RyYXRlZ3kuIEZpcnN0LCB3ZSdyZSBnb2luZyB0byBleHBsb3JlIHNvbWUgDQptZXRob2RzIGZvciBtb2RlbCBjb21wYXJpc29uLiBUaGVuIHdlJ3JlIGdvaW5nIHRvIGxvb2sgYXQgc29tZSBlbXBpcmljYWwgDQp0ZXN0cyB3ZSBjYW4gYXBwbHkgdG8gbW9kZWxzLiBUaGVuIHdlJ2xsIGxvb2sgYXQgbWV0aG9kcyBmb3IgZGVtb25zdHJhdGluZyBhbmQgDQpjb21tdW5pY2F0aW5nIG1vZGVscyB0byB0aGUgd2lkZXIgd29ybGQuIA0KDQojIyBSZXNpZHVhbHMNCg0KSXQgY2FuIGJlIGhlbHBmdWwgdG8gdmlzdWFsbHkgaW5zcGVjdCB0aGUgcmVzaWR1YWxzLCBhcyB3ZSBoYXZlIHNlZW4gcHJldmlvdXNseSwgDQpub3QganVzdCB0byBkaWFnbm9zZSBtaXNmaXQsIGJ1dCB0byBnZXQgYSBzZW5zZSBvZiB0aGUgYWNjdXJhY3kgb2Ygb3VyIG1vZGVsLiANCldlIGNhbiBjcmVhdGUgcmVzaWR1YWwgcGxvdHMgY29tYmluaW5nIHRoZSBmaXRzIGZyb20gYWxsIG9mIHRoZSBtb2RlbHMgcmVsYXRpdmVseSANCmVhc2lseSB3aXRoIGFub3RoZXIgc2V0IG9mIGxvb3BpbmcgY29kZS4gDQoNCg0KYGBge3IgZ2V0cmVzaWR1YWxzZnJvbWVhY2htb2RlbH0NCmJ5X2dyb3VwIDwtIGJ5X2dyb3VwICU+JQ0KICBtdXRhdGUoDQogICAgcmVzaWRzID0gbWFwMihkYXRhLCBtb2RlbCwgYWRkX3Jlc2lkdWFscyksDQogICAgcHJlZHMgPSBtYXAyKGRhdGEsIG1vZGVsLCBhZGRfcHJlZGljdGlvbnMpDQogICkNCmJ5X2dyb3VwDQoNCiMgdW5uZXN0IGV4cGFuZHMgdGhlIGRhdGEgb3V0IGZyb20gYSBjb2x1bW4NCnBsb3RkZiA8LSBsZWZ0X2pvaW4odW5uZXN0KGJ5X2dyb3VwLCByZXNpZHMpLA0KICAgICAgICAgICAgICAgICAgICB1bm5lc3QoYnlfZ3JvdXAsIHByZWRzKSkNCg0KIyBSZXNpZHVhbCBwbG90DQoNCnBsb3RkZiAlPiUNCiAgZ2dwbG90KGFlcyh4ID0gcHJlZCwgeSA9IHJlc2lkKSkgKw0KICBnZW9tX3BvaW50KGFscGhhID0gSSgwLjIpKSArIHRoZW1lX2J3KCkgKw0KICBnZW9tX3Ntb290aCgpDQoNCiMgUmVzaWR1YWwgc2VnbWVudCBwbG90DQpwbG90ZGYgJT4lIHRvcF9uKDIwMCkgJT4lIA0KICBnZ3Bsb3QoYWVzKHggPSBzczEsIHhlbmQgPSBzczEsIHkgPSBwcmVkLCB5ZW5kID0gc3MxKSkgKw0KICBnZW9tX3NlZ21lbnQoYWxwaGEgPSBJKDAuMikpICsgdGhlbWVfYncoKSArDQogIGdlb21fc21vb3RoKCkNCg0KIyBSZXNpZHVhbCBhcnJvdyBwbG90DQpwbG90ZGYgJT4lIHNhbXBsZV9uKDUwKSAlPiUgDQogIGdncGxvdChhZXMoeCA9IHNzMSwgeGVuZCA9IHNzMSwgeSA9IHNzMiwgeWVuZCA9IHByZWQpKSArIA0KICAgIGdlb21fc2VnbWVudChhcnJvdyA9IGFycm93KGFuZ2xlID0gMTUsIGxlbmd0aCA9IHVuaXQoMC4xLCAiaW5jaGVzIikpLCBsaW5lZW5kID0gImJ1dHQiKSAgKyANCiAgICBnZW9tX3Ntb290aChhZXMoeCA9IHNzMSwgeSA9IHNzMiksIHNlPUZBTFNFKSArIA0KICAgICAgY293cGxvdDo6dGhlbWVfY293cGxvdChmb250X3NpemUgPSAxNikgKyANCiAgZ2VvbV9wb2ludChhZXMoeCA9IHNzMSwgeSA9IHNzMikpICsgDQogIGxhYnModGl0bGUgPSAiUmVzaWR1YWxzIFBsb3R0ZWQgYnkgT2JzZXJ2ZWQgVmFsdWVzIG9mIFByZS1UZXN0IiwgDQogICAgICAgY2FwdGlvbiA9ICJBcnJvd3Mgc2hvdyBwcmVkaWN0ZWQgdmFsdWUgY29tcGFyZWQgdG8gb2JzZXJ2ZWQgdmFsdWUgcG9pbnQuIiwgDQogICAgICAgeCA9ICJQcmUtVGVzdCIsIHkgPSAiUG9zdHRlc3QiLCANCiAgICAgICBzdWJ0aXRsZSA9ICJSZXNpZHVhbHMgZnJvbSA1MCB5ZWFyLXN1YmplY3QtZ3JhZGUgbW9kZWxzIG9mIHByZXRlc3Qgb24gcG9zdHRlc3QiKQ0KYGBgDQoNCldlIG5vdGljZSB0aGF0IGV2ZW4gd2hlbiB3ZSBoYXZlIGZpdCA1MCBtb2RlbHMgdG8gZGlmZmVyZW50IHNldHMgb2YgdGhlIGRhdGEgDQpvdXIgbW9kZWxzIGV4aGliaXQgdGhlIGZ1bm5lbGluZyBhcyB0aGUgcHJlZGljdGVkIHZhbHVlIGluY3JlYXNlcy4gV2UgYWxyZWFkeSANCmtub3cgdGhpcyBpcyBhIHN5bXB0b20gb2YgbW9kZWwgbWlzc3BlY2lmaWNhdGlvbi4gDQoNCkZ1cnRoZXIgaW5zcGVjdGlvbiBvZiB0aGUgcmVzaWR1YWxzIGlzIGhlbHBmdWwgaGVyZSAtIHNvIGxldCdzIGxvb2sgYXQgc29tZSANCmZvcm1hbCB3YXlzIG9mIGlkZW50aWZ5aW5nIG91dGxpZXJzLCBoaWdoIGxldmVyYWdlLCBhbmQgaW5mbHVlbnRpYWwgb2JzZXJ2YXRpb25zIA0KbmV4dC4gDQoNCg0KIyMgT3V0bGllcnMsIEhpZ2ggTGV2ZXJhZ2UsIGFuZCBJbmZsdWVudGlhbCBPYnNlcnZhdGlvbnMNCg0KQXMgd2UgaGF2ZSBtZW50aW9uZWQsIE9MUyByZWdyZXNzaW9uIG1vZGVscyBjYW4gYmUgc2Vuc2l0aXZlIHRvIG91dGxpZXJzLiBUaGVyZSANCmFyZSBhIG51bWJlciBvZiBkaWFnbm9zdGljIHRlY2huaXF1ZXMgdGhhdCBjYW4gYmUgdXNlZCB0byBpZGVudGlmeSBvdXRsaWVycy4gDQoNCkFsbCBxdW90ZXMgYmVsb3cgYW5kIGV4Y2VsbGVudCBmdXJ0aGVyIGRldGFpbCBvbiByZWdyZXNzaW9uIGRpYWdub3N0aWNzIGlzIA0KYXZhaWxhYmxlIHRocm91Z2ggdGhlIGZyZWUsIG9ubGluZSwgUGVubiBTdGF0ZSBjb3Vyc2Ugb24gcmVncmVzc2lvbiAtIGF2YWlsYWJsZSANCmhlcmU6IGh0dHBzOi8vb25saW5lY291cnNlcy5zY2llbmNlLnBzdS5lZHUvc3RhdDUwMS9ub2RlLzMzOA0KDQpBIGNvbW1vbiBtZXRob2QgdG8gbWVhc3VyZSBwb3RlbnRpYWwgb3V0bGllcnMgaXMgdG8gY29tcHV0ZSB0aGUgImhhdCB2YWx1ZXMiIG9yIA0KImxldmVyYWdlIiBmb3IgZWFjaCBvYnNlcnZhdGlvbiBpbiBhIHJlZ3Jlc3Npb24gbW9kZWwuICANCg0KTGV2ZXJhZ2VzIGFyZSBkZWZpbmVkIGFzOiANCg0KPiB0aGUgbGV2ZXJhZ2UsICRoX3tpaX0kLCBxdWFudGlmaWVzIHRoZSBpbmZsdWVuY2UgdGhhdCB0aGUgb2JzZXJ2ZWQgcmVzcG9uc2UNCiR5X3tpfSQgaGFzIG9uIGl0cyBwcmVkaWN0ZWQgdmFsdWUgJFxoYXQgeV97aX0kLiBUaGF0IGlzLCBpZiAkaF97aWl9JCBpcyBzbWFsbCwNCnRoZW4gdGhlIG9ic2VydmVkIHJlc3BvbnNlICR5X3tpfSQgcGxheXMgb25seSBhIHNtYWxsIHJvbGUgaW4gdGhlIHZhbHVlIG9mIHRoZQ0KcHJlZGljdGVkIHJlc3BvbnNlICRcaGF0IHlfe2l9JC4gT24gdGhlIG90aGVyIGhhbmQsIGlmICRoX3tpaX0kIGlzIGxhcmdlLCB0aGVuDQp0aGUgb2JzZXJ2ZWQgcmVzcG9uc2UgJHlfe2l9IHBsYXlzIGEgbGFyZ2Ugcm9sZSBpbiB0aGUgdmFsdWUgb2YgdGhlIHByZWRpY3RlZA0KcmVzcG9uc2UgJFxoYXQgeV97aX0kLiBJdCdzIGZvciB0aGlzIHJlYXNvbiB0aGF0IHRoZSAkaF97aWl9JCBhcmUgY2FsbGVkIHRoZQ0KImxldmVyYWdlcy4iDQoNCkxldmVyYWdlcyBjYW4gYmUgaW50ZXJwcmV0ZWQgYXMgdGhlIGRpc3RhbmNlIGJldHdlZW4gdGhlIHggdmFsdWVzIG9mIGFuDQpvYnNlcnZhdGlvbiBhbmQgdGhlIG1lYW4gb2YgdGhlIHggdmFsdWVzIGZvciBhbGwgcmVtYWluaW5nIGRhdGEgcG9pbnRzLg0KTGV2ZXJhZ2VzIGhhdmUgcmFuZ2UgZnJvbSAwIHRvIDEgYW5kIHRoZSBzdW0gb2YgYWxsIG9mIHRoZSBsZXZlcmFnZXMgaXMgZXF1YWwgdG8NCnRoZSB0b3RhbCBudW1iZXIgb2YgcGFyYW1ldGVycyAocmVncmVzc2lvbiBjb2VmZmljaWVudHMsIGluY2x1ZGluZyB0aGUNCmludGVyY2VwdCkgaW4gdGhlIG1vZGVsLg0KDQpBIHdvcmQgb2YgY2F1dGlvbiEgUmVtZW1iZXIsIGEgZGF0YSBwb2ludCBoYXMgbGFyZ2UgaW5mbHVlbmNlIG9ubHkgaWYgaXQgYWZmZWN0cw0KdGhlIGVzdGltYXRlZCByZWdyZXNzaW9uIGZ1bmN0aW9uLiBMZXZlcmFnZXMgb25seSB0YWtlIGludG8gYWNjb3VudA0KdGhlIGV4dHJlbWVuZXNzIG9mIHRoZSB4IHZhbHVlcywgYnV0IGEgaGlnaCBsZXZlcmFnZSBvYnNlcnZhdGlvbiBtYXkgb3IgbWF5IG5vdA0KYWN0dWFsbHkgYmUgaW5mbHVlbnRpYWwuDQoNClRoZXJlIGlzIHN1Y2ggYW4gaW1wb3J0YW50IGRpc3RpbmN0aW9uIGJldHdlZW4gYSBkYXRhIHBvaW50IHRoYXQgaGFzIGhpZ2gNCmxldmVyYWdlIGFuZCBvbmUgdGhhdCBoYXMgaGlnaCBpbmZsdWVuY2UgdGhhdCBpdCBpcyB3b3J0aCBzYXlpbmcgaXQgb25lIG1vcmUNCnRpbWU6DQoNCj4gVGhlIGxldmVyYWdlIG1lcmVseSBxdWFudGlmaWVzIHRoZSBwb3RlbnRpYWwgZm9yIGEgZGF0YSBwb2ludCB0byBleGVydCBzdHJvbmcNCmluZmx1ZW5jZSBvbiB0aGUgcmVncmVzc2lvbiBhbmFseXNpcy4gVGhlIGxldmVyYWdlIGRlcGVuZHMgb25seSBvbiB0aGUgcHJlZGljdG9yDQp2YWx1ZXMuIFdoZXRoZXIgdGhlIGRhdGEgcG9pbnQgaXMgaW5mbHVlbnRpYWwgb3Igbm90IGFsc28gZGVwZW5kcyBvbiB0aGUNCm9ic2VydmVkIHZhbHVlIG9mIHRoZSByZXBvbnNlICR5X3tpfSQuDQoNCiMjIyBIaWdoIExldmVyYWdlDQoNClRvIGJldHRlciB1bmRlcnN0YW5kIG91ciBkYXRhLCBsZXQncyBzdGFydCBieSBsb29raW5nIGF0IG9ic2VydmF0aW9ucyB3aXRoIGEgDQpoaWdoIHBvdGVudGlhbCB0byBpbmZsdWVuY2Ugb3VyIG1vZGVsIC0tIGhpZ2gtbGV2ZXJhZ2Ugb2JzZXJ2YXRpb25zLiBUbyBjb21wdXRlIA0KdGhlIGxldmVyYWdlcywgd2UgY2FuIHVzZSB0aGUgYGF1Z21lbnQoKWAgZnVuY3Rpb24gaW4gdGhlIGBicm9vbWAgcGFja2FnZSwgd2hpY2ggDQphZGRzIG9ic2VydmF0aW9uLWxldmVsIHJlZ3Jlc3Npb24gZGlhZ25vc3RpY3MgdG8gb3VyIG9yaWdpbmFsIGRhdGEgZnJhbWUgZm9yIA0KY29udmVuaWVudCBhbmFseXNpcyBhbmQgcGxvdHRpbmcuIA0KDQpGb3Igc2ltcGxpY2l0eSwgd2UnbGwgZml0IG9uZSBnbG9iYWwgbW9kZWwgYW5kIHdvcmsgd2l0aCBpdC4gQXQgdGhlIGVuZCwgd2UnbGwgDQpyZXR1cm4gdG8gaG93IHRvIGFwcGx5IHRoaXMgYW5hbHlzaXMgdG8gc3VibW9kZWxzLiANCg0KDQpgYGB7cn0NCmxpYnJhcnkoYnJvb20pICMgUiBwYWNrYWdlIGZvciB3b3JraW5nIHdpdGggcmVncmVzc2lvbiBtb2RlbHMgc21vb3RobHkNCiMgQ3JlYXRlIGEgbmV3IGRhdGFzZXQgd2l0aCBhdWdtZW50ZWQgdmFsdWVzDQptMSA8LSBsbShzczIgfiBzczEgKyBuMSArIHN1YmplY3QsIGRhdGEgPSBmdWxsX2RhdGEpDQpzY2hfc2NvcmVfbTEgPC0gYXVnbWVudChtMSkgIyBnZW5lcmF0ZSB0aGUgbGV2ZXJhZ2VzIGFuZCBtb3JlIGZvciBlYWNoIHJvdyBpbiBkYXRhDQpuYW1lcyhzY2hfc2NvcmVfbTEpDQoNCmBgYA0KDQpXZSBoYXZlIG91ciBvcmlnaW5hbCBvdXRjb21lIHZhcmlhYmxlLCBwcmVkaWN0b3IgdmFyaWFibGUsIG91ciBtb2RlbCBmaXRzLCB0aGUgDQpzdGFuZGFyZCBlcnJvciBvZiBvdXIgbW9kZWwgZml0cywgdGhlIHJlc2lkdWFscywgYW5kIHRoZW4gYSBzZXJpZXMgb2YgZGlhZ25vc3RpYyANCnZhcmlhYmxlczogYC5oYXRgLCBgLnNpZ21hYCwgYC5jb29rc2RgLCBhbmQgYC5zdGQucmVzaWRgLiBGb3Igbm93LCB3ZSB3aWxsIGZvY3VzIA0Kb24gdGhlIGAuaGF0YCB2YXJpYWJsZSBhcyB0aGlzIGlzIHRoZSBtZWFzdXJlIG9mIGxldmVyYWdlIGZvciBlYWNoIG9ic2VydmF0aW9uLiANCk9uZSBjb252ZW50aW9uYWwgcnVsZSBvZiB0aHVtYiBpcyB0aGF0IGFuIG9ic2VydmF0aW9uIHdpdGggYSBoYXQgdmFsdWUgZ3JlYXRlciANCnRoYW4gMyB0aW1lcyB0aGUgbWVhbiBoYXQgdmFsdWUgaXMgcG90ZW50aWFsbHkgaW5mbHVlbnRpYWwuIExldCdzIGNvbXB1dGUgYSANCnRhYmxlIG9mIGhvdyBtYW55IG9ic2VydmF0aW9ucyBtZWV0IHRoaXMgY3JpdGVyaWEuIA0KDQoNCmBgYHtyfQ0KdGFibGUoc2NoX3Njb3JlX20xJC5oYXQgPiBtZWFuKHNjaF9zY29yZV9tMSQuaGF0KSozKQ0KYGBgDQoNCldlIGhhdmUgYSBiaWcgZGF0YXNldCwgc28gd2hpbGUgMzYyIGlzIGEgbGFyZ2UgbnVtYmVyIG9mIG9ic2VydmF0aW9ucywNCnByb3BvcnRpb25hbGx5IGl0IGlzIGZhaXJseSBzbWFsbC4gVGhpcyBpcyBhIHR5cGljYWwgcmVzdWx0IGZvciBhIGRhdGFzZXQgdGhpcw0Kc2l6ZSBhbmQgYSBtb2RlbCBvZiB0aGlzIG5hdHVyZS4NCg0KKipCb251cyBxdWVzdGlvbjogSG93IGNhbiB5b3UgdmVyaWZ5IHRoYXQgdGhpcyBhbW91bnQgaXMgdHlwaWNhbD8qKg0KDQpSZW1lbWJlciB0aGF0IG91ciByZXNlYXJjaCBxdWVzdGlvbiBpcyBhcm91bmQgdGhlIG91dGxpZXJzIHRob3VnaCAtIHdlIHdhbnQgDQp0aGUgbW9kZWwgdG8gaWRlbnRpZnkgbWVhbmluZ2Z1bCBvdXRsaWVycywgc28gcGF5aW5nIGNhcmVmdWwgYXR0ZW50aW9uIHRvIHRoZSANCmF0dHJpYnV0ZXMgb2YgdGhlc2Ugb2JzZXJ2YXRpb25zIGlzIGltcG9ydGFudC4gDQoNCldlIGNhbiBnZXQgYSBsaXR0bGUgbW9yZSBkZXRhaWwgYnkgY2FsY3VsYXRpbmcgdGhlIHJhdGlvIG9mIGVhY2ggaGF0IHZhbHVlIHRvIA0KdGhlIG1lYW4gb2YgYWxsIHRoZSBoYXQgdmFsdWVzLiBUaGUgY29kZSBiZWxvdyByb3VuZHMgdGhpcyByYXRpbyB0byB0aGUgbmVhcmVzdCANCnRlbnRoIGFuZCBwcmVzZW50cyBpdCBpbiBhIHRhYmxlLCBhbGxvd2luZyB1cyB0byBnZXQgYSBmdWxsZXIgcGljdHVyZSBvZiB0aGUgDQp3aG9sZSBkaXN0cmlidXRpb24gb2YgaGF0IHZhbHVlIC0gcmVtZW1iZXIgdGhhdCB2YWx1ZXMgZ3JlYXRlciB0aGFuIDN4IHRoZSANCm1lYW4gYXJlIGNvbnNpZGVyZWQgaGlnaCBsZXZlcmFnZS4gDQoNCmBgYHtyfQ0KIyBFeHBhbmRlZCBjb2RlIHRvIG1ha2UgaXQgZWFzaWVyIHRvIHJlYWQ6DQprbml0cjo6a2FibGUoICMgdG8gbWFrZSBhIHByZXR0eSB0YWJsZSBpbiBvdXIgZG9jdW1lbnQNCiAgdGFibGUoICMgY29tcHV0ZSBhIHRhYmxlDQogICAgcm91bmQoICMgcm91bmQgdGhlIHJlc3VsdCwgaGVyZSB0byB0aGUgdGVudGhzICgxKQ0KICAgICAgc2NoX3Njb3JlX20xJC5oYXQvbWVhbihzY2hfc2NvcmVfbTEkLmhhdCksIDENCiAgICAgICkNCiAgICApDQogICkNCmBgYA0KDQoNCk5vdyB0aGF0IHdlIGhhdmUgYSBzZW5zZSBvZiB0aGUgZGlzdHJpYnV0aW9uIG9mIGhpZ2ggbGV2ZXJhZ2Ugb2JzZXJ2YXRpb25zIGFuZCANCmhvdyBtYW55IHdlIGhhdmUgKGhvdyB3b3VsZCB5b3UgZGVzY3JpYmUgdGhpcyBkaXN0cmlidXRpb24/KSAtIGl0J3MgdGltZSB0byANCmxvb2sgYXQgdGhlIGltcGFjdCB0aGV5IGhhdmUgb24gdGhlIHJlZ3Jlc3Npb24gbW9kZWwgaXRzZWxmIGJ5IGRpcmVjdGx5IG1lYXN1cmluZyANCnRoZWlyIGluZmx1ZW5jZS4gDQoNCiMjIyMgSW5mbHVlbnRpYWwgT2JzZXJ2YXRpb25zDQoNCk9uY2Ugd2UgaGF2ZSBpZGVudGlmaWVkIG91ciBoaWdoIGxldmVyYWdlIG9ic2VydmF0aW9ucywgd2UgY2FuIGNvbWJpbmUgdGhpcyANCmluZm9ybWF0aW9uIHdpdGggaW5mb3JtYXRpb24gYWJvdXQgdGhlIGluZmx1ZW5jZSBvZiBvYnNlcnZhdGlvbnMgLSBpbmNvcnBvcmF0aW5nIA0KaW5mb3JtYXRpb24gYWJvdXQgdGhlIGRlcGVuZGVudCB2YXJpYWJsZSBhcyB3ZWxsLiBBIHF1aWNrIGZpcnN0IGxvb2sgYXQgdGhpcyBpcyANCnRvIGxvb2sgYXQgdGhlIGhhdCB2YWx1ZXMgdGhhdCBhbHNvIGhhdmUgbGFyZ2UgKG9yIHNtYWxsKSB2YWx1ZXMgb2YgdGhlIG91dGNvbWUgdmFyaWFibGUuIA0KVGhlc2Ugb2JzZXJ2YXRpb25zLCBtYXRoZW1hdGljYWxseSwgd2lsbCBleGVydCBleHRyYSBpbmZsdWVuY2Ugb24gb3VyIGVzdGltYXRlIA0Kb2YgYmV0YSwgYW5kIGFyZSB0aHVzIGluZmx1ZW50aWFsIG9uIG91ciBtb2RlbC4gIA0KDQpUaGlzIGNvZGUgc2ltdWxhdGVzIGEgcGVyZmVjdGx5IHNwZWNpZmllZCByZWdyZXNzaW9uIG1vZGVsLiANCg0KYGBge3J9DQojIEdlbmVyYXRlIGFuIGlkZWFsIGhpZ2ggbGV2ZXJhZ2UgcGxvdA0Kc2V0LnNlZWQoNTI1MjMpDQojIFNldCB2YWx1ZXMgdG8gc2ltdWxhdGUgdGhlIGRhdGENCk4gPC0gMTUwMDANCngxIDwtIHJub3JtKE4pDQpiMSA8LSBybm9ybShOLCAwLjgsIDAuMjUpDQojIGExIDwtIHJiaW5vbShOLCBzaXplID0gMSwgcHJvYiA9IDAuNykNCnkgPC0gNiArIGIxICogeDEgKyBybm9ybShOLCAwLCAwLjUpDQptX2V4IDwtIGxtKHkgfiB4MSkNCnBsb3RkZiA8LSBhdWdtZW50KG1fZXgpDQpnZ3Bsb3QocGxvdGRmLCBhZXMoeCA9IHksIHkgPSAuaGF0KSkgKyBnZW9tX3BvaW50KGFscGhhID0gMS81KSArIHRoZW1lX2J3KCkgKyANCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gMyAqIG1lYW4ocGxvdGRmJC5oYXQpKQ0KYGBgDQoNCkluIGEgd2VsbC1zcGVjaWZpZWQgcmVncmVzc2lvbiBtb2RlbCB0aGUgaGlnaCBsZXZlcmFnZSBvYnNlcnZhdGlvbnMgd2lsbCBiZSANCmV2ZW5seSBiYWxhbmNlZCBiZXR3ZWVuIGhpZ2ggYW5kIGxvdyB2YWx1ZXMgb2YgdGhlIG91dGNvbWUsIGFzIGluIHRoZSBwbG90IGFib3ZlLiANClRoaW5rIGFib3V0IHRoZSBwb2ludHMgYWJvdmUgdGhlIGhpZ2gtbGV2ZXJhZ2UgaG9yaXpvbnRhbCBsaW5lIGFzIGJhbGFuY2luZyBhZ2FpbnN0IA0Kb25lIGFub3RoZXIgLS0gdG9vIG1hbnkgdG8gb25lIHNpZGUgb3IgYW5vdGhlciBhbmQgdGhlIHJlc3VsdHMgb2YgdGhlIG1vZGVsIA0KbWF5IGJlIHNrZXdlZCBhbmQgYXQgcmlzayBvZiBiZWluZyBiaWFzZWQgYnkgb3V0bGllciANCm9ic2VydmF0aW9ucy4gDQoNCkxldCdzIHNlZSBob3cgdGhlIG1vZGVsIHdlIGZpdCBsb29rcyBieSB0aGlzIG1ldHJpYzoNCg0KYGBge3J9DQpnZ3Bsb3Qoc2NoX3Njb3JlX20xLCBhZXMoeCA9IHNzMiwgeSA9IC5oYXQpKSArIA0KICBnZW9tX3BvaW50KGFscGhhID0gMS81KSArIA0KICB0aGVtZV9idygpICsgDQogIGdlb21fc21vb3RoKHNlPUZBTFNFKSArIA0KICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAzICogbWVhbihzY2hfc2NvcmVfbTEkLmhhdCkpICMgZ2VuZXJhdGUgYSByZWZlcmVuY2UgbGluZQ0KDQpgYGANCg0KDQpXZSBkb24ndCBoYXZlIHRvIHJlbHkgb24gdGhlIGV5ZS10ZXN0IGFsb25lLiBGb3J0dW5hdGVseSwgd2UgaGF2ZSBhIGZ1cnRoZXIgc2V0IA0Kb2YgZGlhZ25vc3RpY3MgYXZhaWxhYmxlIHRvIHVzZSB0byBtZWFzdXJlIA0KdGhlIGluZmx1ZW5jZSB0aGF0IGVhY2ggb2JzZXJ2YXRpb24gaGFzIG9uIG91ciBlc3RpbWF0ZXMgb2YgJFxiZXRhJC4gQ29sbGVjdGl2ZWx5IA0Ka25vd24gYXMgaW5mbHVlbmNlIG1lYXN1cmVzLCB0aGVzZSBjb21wdXRlIHRoZSBpbmZsdWVuY2Ugb2YgaW5kaXZpZHVhbCBvYnNlcnZhdGlvbnMgDQpieSBsb29raW5nIGF0IHRoZSBkaWZmZXJlbmNlIGluIHRoZSBtb2RlbCB3aXRob3V0IGVhY2ggb2JzZXJ2YXRpb24sIGNvbXBhcmVkIHRvIA0KdGhlIG1vZGVsIHdpdGggYWxsIG9ic2VydmF0aW9ucy4gVGhpcyBkZWxldGlvbiB0ZWNobmlxdWUgYWxsb3dzIHVzIHRvIGVtcGlyaWNhbGx5IA0KYXNzZXNzIHRoZSBjb250cmlidXRpb24gb2YgZWFjaCBvYnNlcnZhdGlvbiB0byB0aGUgY29lZmZpY2llbnRzIGluIHRoZSBtb2RlbC4gDQpUaGlzIG1ldHJpYyBpcyBjYWxsZWQgYSBgREZCRVRBYC4gVGhlIGBkZmJldGFgIGZvciBlYWNoIG9ic2VydmF0aW9uIGlzIGNhbGN1bGF0ZWQgDQpmb3IgZWFjaCByZWdyZXNzb3IgaW4gb3VyIGRhdGFzZXQsIHNvIHdpdGggdGhyZWUgcmVncmVzc29ycyBhbmQgYW4gaW50ZXJjZXB0IHdlIA0Kd2lsbCBnZXQgNCBERkJFVEEgbWVhc3VyZXMuIA0KDQpSIHByb3ZpZGVzIHVzIGEgY29udmVuaWVuY2UgZnVuY3Rpb24gdGhhdCBjb21wdXRlcyB0aGUgZGVsZXRpb24gbWVhc3VyZXMgZm9yIHVzLiANClRoZSBgaW5mbHVlbmNlLm1lYXN1cmVzKClgIHJldHVybnMgYSBsaXN0IG9mIG1hdHJpY2VzIGNvbXB1dGluZyB0aGVzZSBmZWF0dXJlcy4gDQoNCmBgYHtyIGluZmx1ZW5jZUhpc3RvZ3JhbX0NCm0xX2luZiA8LSBpbmZsdWVuY2UubWVhc3VyZXMobTEpDQpoZWFkKG0xX2luZiRpbmZtYXQpDQpgYGANCg0KVGhpcyBmb3JtYXQgaXNuJ3QgcGFydGljdWxhcmx5IGhlbHBmdWwgdG8gdXMgLSBpdCBpcyBoYXJkIHRvIGludGVycHJldCB0aGVzZSBudW1iZXJzLiANCkxldCdzIGZvY3VzIG9uIHRoZSBjb2VmZmljaWVudCBmb3IgYHNzMWAgZ2l2ZW4gYnkgdGhlIGNvbHVtbiBgZGZiLnNzMWAuIFRoaXMgDQpyZXByZXNlbnRzIHRoZSBjaGFuZ2UgaW4gdGhlIGNvZWZmaWNpZW50IHdpdGggdGhpcyBvYnNlcnZhdGlvbiBleGNsdWRlZCwgYnV0IHRoZSANCmNoYW5nZSBpcyBzY2FsZWQgYnkgdGhlIHN0YW5kYXJkIGVycm9yIG9mIHRoZSBjb2VmZmljaWVudCBpdHNlbGYuIExldCdzIGRvIGEgDQpxdWljayB3b3JrZWQgZXhhbXBsZSB0byB1bmRlcnN0YW5kIGhvdyB0aGUgc2NhbGUgb2YgdGhpcyANCmltcG9ydGFudCBtZWFzdXJlIHdvcmtzLiANCg0KRmlyc3QsIGxldCdzIHJlbWluZCBvdXJzZWx2ZXMgb2YgdGhlIHN0YW5kYXJkIGVycm9yIGFuZCBlc3RpbWF0ZSBvZiBvdXIgY29lZmZpY2llbnQgDQpmb3IgYHNzMWAuIA0KDQpgYGB7cn0NCnN1bW1hcnkobTEpDQpgYGANCg0KVGhlIGNvZWZmaWNpZW50IGlzIDAuODUyODkzIGFuZCB0aGUgc3RhbmRhcmQgZXJyb3IgaXMgfjAuMDAzLiBOb3csIGxldCdzIGxvb2sgYXQgDQp0aGUgbGFyZ2VzdCBhbmQgc21hbGxlc3QgdmFsdWVzIG9mIERGQkVUQVM6IA0KDQpgYGB7cn0NCm1heChtMV9pbmYkaW5mbWF0WywgMl0gKQ0KbWluKG0xX2luZiRpbmZtYXRbLCAyXSApDQpgYGANCg0KVG8gc2VlIHRoZSBjaGFuZ2UgaW4gdGhlIHNsb3BlIG9uIGBzczFgIHRoYXQgcmVzdWx0cywgd2UgbmVlZCB0byBkbyBzb21lIHF1aWNrIA0KbWF0aCAtLSBmaXJzdCB3ZSAidW5zdGFuZGFyZGl6ZSIgdGhlIGVzdGltYXRlIGJ5IG11bHRpcGx5aW5nIGJ5IHRoZSBzdGFuZGFyZCANCmVycm9yOiANCg0KYGBge3J9DQptYXgobTFfaW5mJGluZm1hdFssIDJdICkgKiAwLjAwMw0KbWluKG0xX2luZiRpbmZtYXRbLCAyXSApICogMC4wMDMNCmBgYA0KDQpBbmQgdGhlbiB3ZSBhZGQgdGhpcyB0byB0aGUgY29lZmZpY2llbnQgdG8gc2VlIGhvdyBtdWNoIHRoZSBjb2VmZmljaWVudCB3aWxsIA0KY2hhbmdlOiANCg0KYGBge3J9DQoobWF4KG0xX2luZiRpbmZtYXRbLCAyXSApICogMC4wMDMpICsgMC44NTI4OTMNCihtaW4obTFfaW5mJGluZm1hdFssIDJdICkgKiAwLjAwMykgKyAwLjg1Mjg5Mw0KYGBgDQoNClRoZSBjaGFuZ2UgaXMgbm90IHZlcnkgbXVjaCBmb3Igb3VyIGJpZ2dlc3QgYW5kIHNtYWxsZXN0IHZhbHVlcywgYnV0IHJlbWVtYmVyIHRoYXQgdGhpcyBpcyANCmJlY2F1c2UgdGhlcmUgYXJlID4gMTUsMDAwIG9ic2VydmF0aW9ucyBpbiBvdXIgZGF0YSwgc28gd2UgbmVlZCB0byBmb2N1cyBvbiB0aGUgDQppbXBhY3Qgb2Ygb2JzZXJ2YXRpb25zIHJlbGF0aXZlIHRvIHRoZSBhdmVyYWdlIG9ic2VydmF0aW9uLCBub3QgdGhlaXIgaW1wYWN0IA0KYWxvbmUuIEEgY29tbW9uIHJ1bGUgb2YgdGh1bWIgZm9yIGlkZW50aWZ5aW5nIGxhcmdlIERGQkVUQSB2YWx1ZXMgaXMgYW55IHZhbHVlIA0KbGFyZ2VyIHRoYW4gMiAvIHNxcnQobikgd2hlcmUgbiBpcyB0aGUgbnVtYmVyIG9mIHJvd3MgaW4gb3VyIHNhbXBsZS4gDQoNCkluIHRoaXMgY2FzZSwgYW55IERGQkVUQSB3aXRoIGFuIGFic29sdXRlIHZhbHVlIGdyZWF0ZXIgdGhhbjogDQoNCmBgYHtyfQ0KMiAvIHNxcnQobnJvdyhtMSRtb2RlbCkpDQpgYGANCg0KTGV0J3Mgc2VlIGhvdyBtYW55IG9mIHRoZXNlIHdlIGhhdmU6IA0KDQpgYGB7cn0NCnRhYmxlKGFicyhtMV9pbmYkaW5mbWF0WywgMl0gKSA+ICgyL3NxcnQobnJvdyhtMSRtb2RlbCkpKSkNCmBgYA0KDQpOb3csIGZvciB1cywgdGhlIERGQkVUQXMgYmVjb21lIGp1c3QgbGlrZSBhbnkgb3RoZXIgZGF0YSBwb2ludCwgYW5kIHdlIGNhbiANCnVzZSBhIHZhcmlldHkgb2YgRURBIHRlY2huaXF1ZXMgdG8gZXhwbG9yZSBoeXBvdGhlc2VzIGFib3V0IHdoaWNoIG9ic2VydmF0aW9ucyANCnJlcHJlc2VudCB0aGVzZSBvdXRsaWVyIHBvaW50cy4gSGVyZSdzIG9uZSBleGFtcGxlLCB3aGVyZSB3ZSBsb29rIGF0IHZhcmlhYmxlcyANCmluIHRoZSBtb2RlbCBhbmQgcGxvdCB0aGUgREZCRVRBcyBhZ2FpbnN0IHRoZW0gLS0gdGhpcyBzaG93cyB0aGUgcmVsYXRpb25zaGlwIA0KYmV0d2VlbiB0aGUgdmFsdWUgb2YgYHNzMWAsIGl0J3MgREZCRVRBLCBhbmQgaG93IHRoYXQgcmVsYXRpb25zaGlwIGlzIGRpZmZlcmVudCANCmFjcm9zcyByZWFkaW5nIGFuZCBtYXRoLiANCg0KYGBge3J9DQpwbG90ZGYgPC0gc2NoX3Njb3JlX20xDQpwbG90ZGYkZGZiZXRhX3NzMSA8LSBtMV9pbmYkaW5mbWF0WywgMl0NCg0KZ2dwbG90KHBsb3RkZiwgYWVzKHggPSBzczEsIHkgPSBkZmJldGFfc3MxKSkgKyANCiAgZ2VvbV9wb2ludChhbHBoYT1JKDAuMikpICsgDQogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDIvc3FydChucm93KG0xJG1vZGVsKSkpICsgDQogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IC0yL3NxcnQobnJvdyhtMSRtb2RlbCkpKSArIA0KICBmYWNldF93cmFwKH5zdWJqZWN0KSArIA0KICBnZW9tX3J1ZyhhbHBoYT0xLzQsIHBvc2l0aW9uID0gImppdHRlciIpICsNCiAgdGhlbWVfYncoKQ0KDQpgYGANCg0KVGhpcyBpcyBvbmUgd2F5IHRvIGV4cGxvcmUgdGhlIERGQkVUQXMsIGFuZCB3aGF0IGRvZXMgaXQgc2hvdyB1cy4gRmlyc3QsIGl0IA0KY29uZmlybXMgdGhhdCBiaWdnZXIgYW5kIHNtYWxsZXIgdmFsdWVzIG9mIHRoZSB2YXJpYWJsZSBhcmUgbW9yZSBsaWtlbHkgdG8gaGF2ZSANCmEgYmlnZ2VyIGluZmx1ZW5jZSBvbiB0aGUgY29lZmZpY2llbnQuIEJ1dCwgdGhlIHBhdHRlcm4gaXMgdmVyeSBkaWZmZXJlbnQgYmV0d2VlbiANCnRoZSB0d28gc3ViamVjdHMsIGFuZCB0aGUgZGlzdHJpYnV0aW9uIG9mIG91dGxpZXIgdmFsdWVzICh0aG9zZSBvdXRzaWRlIG9mIHRoZSANCmhvcml6b250YWwgbGluZXMpIGFyZSBhbHNvIGRpZmZlcmVudC4gV2hhdCBkb2VzIHRoaXMgZXZpZGVuY2Ugc3VnZ2VzdCBhYm91dCBob3cgDQp5b3UgY291bGQgY2hhbmdlIHRoaXMgbW9kZWw/IA0KDQpJbiB0aGlzIHBsb3QsIHdlIGluY2x1ZGVkIGEgDQpgcnVnYCBlbGVtZW50IG9uIHRoZSBib3R0b20gd2hpY2ggc2hvd3MgZWFjaCBvYnNlcnZlZCB2YWx1ZSBvZiB0aGUgeC1heGlzIGFuZCANCnktYXhpcyBhcyBhIHRpY2suIEhlcmUgd2UgY2FuIHNlZSB0aGF0IHRob3VnaCB0aGUgZGlzdHJpYnV0aW9uIGlzIGJhbGFuY2VkLCANCmV4dHJlbWUgdmFsdWVzIGFyZSBub3QgYmFsYW5jZWQgLSB3aXRoIG1vcmUgZXh0cmVtZSB2YWx1ZXMgcHJlc2VudCBvbiB0aGUgDQpsb3ctZW5kIG9mIHRoZSBzY2FsZSAoPCAtMC4wNSkgdGhhbiBhdCB0aGUgaGlnaCBlbmQgKD4gMC4wNSkuIFRoaXMgbWVhbnMgDQp0aGF0IHRoZXJlIA0KYXJlIG1vcmUgb2JzZXJ2YXRpb25zIHRoYXQgYXJlIHB1bGxpbmcgdGhlIHNsb3BlIG9mIGBzczFgIGRvd253YXJkIHRvIGEgaGlnaCANCmRlZ3JlZSB0aGFuIHVwd2FyZC4gSWYgd2UgZXhjbHVkZWQgYWxsIHZhbHVlcyBvZiBgc3MxYCB0aGF0IGFyZSBhYm92ZSBhbmQgYmVsb3cgDQp0aGUgc2FtZSBjdXQtb2ZmLCB3ZSB3b3VsZCBleHBlY3QgdGhlIGNvZWZmaWNpZW50IG9uIGBzczFgIHRvIGJlIG1vcmUgcG9zaXRpdmUgDQppbiB0aGF0IGNhc2UuIEtub3dpbmcgdGhlIGRpcmVjdGlvbiBvZiB0aGlzIGJpYXMgYW1vbmcgdGhlIGV4dHJlbWUgdmFsdWVzIGluIG91ciANCmRhdGEgaXMgaW1wb3J0YW50IGluIHVuZGVyc3RhbmRpbmcgaG93IHRoZSBtb2RlbCBpcyBiZWhhdmluZyBvbiB0aGlzIHNldCBvZiBkYXRhLiANCg0KDQojIyMjIEZ1cnRoZXIgTWV0cmljcw0KDQpUd28gZnVydGhlciBtZXRyaWNzIHlvdSBtYXkgdXNlIHRvIGlkZW50aWZ5IG91dGxpZXIgb2JzZXJ2YXRpb25zIGFyZSBDb29rJ3MgRCBhbmQgDQpERklUUy4gVGhlc2UgcHJvZHVjZSBhIHNpbWlsYXIgbWV0cmljIHRoYXQgY29tYmluZXMgdGhlIHJlc2lkdWFscyBhbmQgdGhlIGxldmVyYWdlIA0KdG8gbWVhc3VyZSB0cnVlIGluZmx1ZW5jZSBvZiBlYWNoIG9ic2VydmF0aW9uLiBFYWNoIGlzIG9uIGEgZGlmZmVyZW50IHNjYWxlLCBidXQgDQpzaG91bGQgaWRlbnRpZnkgc2ltaWxhciBvYnNlcnZhdGlvbnMuIFNvbWUgcnVsZXMgb2YgdGh1bWIgZm9yIGludGVycHJldGluZyANCnRoZXNlIG1ldHJpY3MgDQooZnJvbSB0aGUgZXhjZWxsZW50IFVDTEEgd2ViYm9vayBvbiByZWdyZXNzaW9uOiBodHRwczovL3N0YXRzLmlkcmUudWNsYS5lZHUvc3RhdGEvd2ViYm9va3MvcmVnL2NoYXB0ZXIyL3N0YXRhLXdlYmJvb2tzcmVncmVzc2lvbndpdGgtc3RhdGFjaGFwdGVyLTItcmVncmVzc2lvbi1kaWFnbm9zdGljcy8pDQoNCi0gQ29vaydzIEQgcmFuZ2VzIGZyb20gMCB0byB2ZXJ5IGxhcmdlIHBvc2l0aXZlIHZhbHVlcy4gVGhlIHRyYWRpdGlvbmFsIGN1dG9mZiANCnBvaW50IGZvciBoaWdoIGluZmx1ZW5jZSBvYnNlcnZhdGlvbnMgaXMgNC9uIHdoZXJlIG4gaXMgdGhlIG51bWJlciBvZiByb3dzIGluIA0KdGhlIHNhbXBsZS4NCi0gREZJVFMgY2FuIHJhbmdlIGZyb20gLSBsYXJnZSB2YWx1ZXMgdG8gcG9zaXRpdmUgbGFyZ2UgdmFsdWVzLiBUaGUgY3V0b2ZmIHBvaW50IA0KZm9yIERGSVRTIGlzIDIgKiBzcXJ0KGsvbikgd2hlcmUgayBpcyB0aGUgbnVtYmVyIG9mIHByZWRpY3RvcnMgaW4gb3VyIG1vZGVsLiANCg0KWW91IGNhbiBhcHBseSBhIHNpbWlsYXIgYW5hbHlzaXMgd2UganVzdCBkaWQgZm9yIHRoZSBERkJFVEFzIHRvIGV4cGxvcmUgdGhlc2UuIA0KTGV0J3MgcXVpY2tseSBnbyB0aHJvdWdoIGFuIGFuYWx5c2lzIHdoZXJlIHdlIGxvb2sgYXQgdGhlIHJlbGF0aW9uc2hpcCBiZXR3ZWVuIA0KaW5mbHVlbnRpYWwgb2JzZXJ2YXRpb25zIGFuZCB2YXJpYWJsZXMgdGhhdCBhcmUgbm90IGluIG91ciBtb2RlbC4gTGV0J3MgdXNlIA0KREZJVFMgYmVjYXVzZSBpdCBrZWVwcyB0aGUgZGlyZWN0aW9uYWxpdHkgaW5mb3JtYXRpb24gKHdoZXRoZXIgdGhlIGluZmx1ZW5jZSBpcyANCnBvc2l0aXZlIG9yIG5lZ2F0aXZlKS4gDQoNCmBgYHtyfQ0KcGxvdGRmIDwtIGZ1bGxfZGF0YQ0KcGxvdGRmJGRmZml0cyA8LSBkZmZpdHMobTEpDQoNCiMgbGltaXRzIGZvciBkZmZpdHMNCmRmZml0c19saW1pdCA8LSAyICogc3FydChsZW5ndGgoY29lZihtMSkpIC8gbnJvdyhtMSRtb2RlbCkpDQoNCmdncGxvdChwbG90ZGYpICsgYWVzKHggPSBzczIsIHkgPSBkZmZpdHMpICsgDQogICAgZ2VvbV9wb2ludChhbHBoYSA9IDEvNCkgKyANCiAgICBmYWNldF9ncmlkKHN1YmplY3QgfiBncmFkZSkgKyANCiAgICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSBkZmZpdHNfbGltaXQpICsgDQogICAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gLWRmZml0c19saW1pdCkgKyANCiAgICB0aGVtZV9idygpDQoNCmBgYA0KDQpIZXJlIHdlIGhhdmUgYnJvdWdodCBpbiBncmFkZSBsZXZlbCwgYSB2YXJpYWJsZSBvdXIgb3JpZ2luYWwgbW9kZWwgbGVmdCBvdXQuIA0KTm93IHdlIHNlZSB0aGUgbW90aXZhdGlvbiBmb3Igc3BsaXR0aW5nIHRoZSBtb2RlbCBieSBzYW1wbGUuIA0KDQpKdXN0IGZvciBmdW4sIGxldCdzIGxvb2sgYXQgb25lIG1vcmU6IA0KDQpgYGB7cn0NCmdncGxvdChwbG90ZGYpICsgYWVzKHggPSBzczIsIHkgPSBkZmZpdHMsIGNvbG9yID0gc2Nob29sX3NpemUpICsgDQogICAgZ2VvbV9wb2ludChhbHBoYSA9IDIvMykgKyANCiAgICBmYWNldF93cmFwKCB+bG9jYWxlX2Rlc2MpICsgDQogIHNjYWxlX2NvbG9yX2JyZXdlcih0eXBlID0gInF1YWwiLCBwYWxldHRlID0gMywgZGlyZWN0aW9uID0gLTEpICsgDQogIGd1aWRlcyhjb2xvciA9IGd1aWRlX2xlZ2VuZChvdmVycmlkZS5hZXMgPSBsaXN0KGFscGhhID0gMSkpKSArDQogICAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gZGZmaXRzX2xpbWl0KSArIA0KICAgIGdlb21faGxpbmUoeWludGVyY2VwdCA9IC1kZmZpdHNfbGltaXQpICsgDQogICAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIsIA0KICAgICAgICAgICAgICAgICAgICAgICBsZWdlbmQua2V5LnNpemUgPSB1bml0KDEyLCAicG9pbnRzIikpDQpgYGANCg0KVGhpcyB0eXBlIG9mIGFuYWx5c2lzIGNhbiBnaXZlIHVzIGluc2lnaHQgaW50byBob3cgb3VyIG1vZGVsIHNob3VsZCBiZSBtb2RpZmllZCANCm9yIGhvdyBvdXIgc2FtcGxlIHNob3VsZCBiZSByZXN0cmljdGVkIHRvIGFjY291bnQgZm9yIHN5c3RlbWF0aWMgdmFyaWF0aW9uIGluIA0KaW5mbHVlbnRpYWwgb2JzZXJ2YXRpb25zIGJ5IGtleSBvYnNlcnZhYmxlIGNoYXJhY3RlcmlzdGljcy4gVHJ5IHNvbWUgb2YgeW91ciANCm93biBhbmFseXNpcyENCg0KIyMgUmVncmVzc2lvbiBEaWFnbm9zdGljcyANCg0KSW4gdGhlIGxlY3R1cmUgd2UgbGVhcm5lZCBhYm91dCB0aGUgdXRpbGl0eSBvZiByZWdyZXNzaW9uIGRpYWdub3N0aWMgdGVzdHMuIFRlc3RzIA0KYXJlIHVzZWZ1bCBiZWNhdXNlIHRoZXkgY2FuIHNjYWxlIHRvIG11bHRpcGxlIG1vZGVscyBhbmQgdGVsbCB1cyB3aGljaCBtb2RlbHMgDQp0byBsb29rIGludG8gZnVydGhlciBhbmQgd2hpY2ggbW9kZWxzIGFyZSBPSy4gV2UgY2FuIGFsc28gbWFrZSBjb21wYXJpc29ucyBiZXR3ZWVuIA0Kc2V0cyBvZiBtdWx0aXBsZSBtb2RlbCBzcGVjaWZpY2F0aW9ucyBieSBjb21wYXJpbmcgaG93IG1hbnkgb2YgdGhlIGZpZnR5IG1vZGVscyANCmZhaWwgdGhlIHRlc3RzIHVuZGVyIGRpZmZlcmVudCBzcGVjaWZpY2F0aW9ucy4gSW4gc2hvcnQsIG1vZGVsIGRpYWdub3N0aWNzIGFyZSANCmEgc2NhbGFibGUgd2F5IHRvIGFzc2VzcyByZWdyZXNzaW9uIG1vZGVscy4gDQoNCldoaWxlIGJhc2UtUiBwcm92aWRlcyBzb21lIGJhc2ljIHJlZ3Jlc3Npb24gZGlhZ25vc3RpYyBmdW5jdGlvbmFsaXR5LCBpdCBpcyBtb3JlIA0KY29uc2lzdGVudGx5IGltcGxlbWVudGVkIHVzaW5nIHRoZSBzdWl0ZSBvZiBmdW5jdGlvbnMgcHJvdmlkZWQgYnkgdGhlIGBsbXRlc3RgIA0KcGFja2FnZS4gVGhlc2UgdGVzdHMgdGFrZSBzb21lIGZvcm0gb2YgZGl2aWRpbmcgdGhlIGRhdGEgb3IgcmVjYXN0aW5nIHRoZSBtb2RlbCwgDQphbmQgdGhlbiBjb21wYXJpbmcgZm9yIHN0YXRpc3RpY2FsbHkgbWVhbmluZ2Z1bCBkaWZmZXJlbmNlcyBpbiB0aGUgZGl2aWRlZCBkYXRhIA0Kb3IgaW4gdGhlIHJlY2FzdGVkIG1vZGVsIHJlc2lkdWFscy4gDQoNCkluIGdlbmVyYWwgdGhlc2UgZnVuY3Rpb25zIGdlbmVyYXRlIGEgdGVzdCBzdGF0aXN0aWMsIGRlZ3JlZXMgb2YgZnJlZWRvbSwgYW5kIA0KdGhlbiBnZW5lcmF0ZSBhIHAtdmFsdWUgYWdhaW5zdCB0aGUgbnVsbC1oeXBvdGhlc2lzIHRoYXQgdGhlIG1vZGVsIGlzIA0KY29ycmVjdGx5IHNwZWNpZmllZC4gSWYgdGhlIHAtdmFsdWUgaXMgbGVzcyB0aGFuIGEgY3JpdGljYWwgdmFsdWUsIHRoZSBkYXRhIA0Kc3VwcG9ydHMgdGhlIGh5cG90aGVzaXMgdGhhdCB0aGUgbW9kZWwgaXMgbm90IHdlbGwtc3BlY2lmaWVkIGFjY29yZGluZyB0byB0aGUgDQpwYXJ0aWN1bGFyIHRlc3QuIA0KDQpUaGlzIGlzIG9uZSBvZiB0aGUgcmFyZSB0aW1lcyB3aGVuIEkgd2lsbCB0ZWxsIHlvdSB0aGF0IHRoZSBwLXZhbHVlIGlzIGEgDQp1c2VmdWwgc3RhdGlzdGljIHRvIGNvbnNpZGVyIG9uIGl0cyBvd24hIEl0J3MganVzdCBtb3JlIGVmZmljaWVudCBpZiB5b3Ugd2FudCANCnRvIHRlc3QgYSBtb2RlbCB3aXRoIG11bHRpcGxlIGRpYWdub3N0aWNzLCBvciBtdWx0aXBsZSBtb2RlbHMgd2l0aCB0aGUgc2FtZSANCmRpYWdub3N0aWMuIFRoZXNlIHRlc3RzIGhhdmUgZGlmZmVyZW50IHRlc3Qgc3RhdGlzdGljcyBhbmQgYXJlIGFsbCBkZXNpZ25lZCANCmZvciBpbnRlcnByZXRhdGlvbiBhcm91bmQgdGhlIHAtdmFsdWUsIHNvIEkgcmVjb21tZW5kIGp1c3QgdXNpbmcgaXQuIFRoZSBjYXZlYXQgDQppcyB0aGF0IHRoaXMgaXMgYXBwbGllZCB3b3JrIGFuZCB0aGUgZGlhZ25vc3RpY3MgYXJlIGludGVuZGVkIHRvIHByb3ZpZGUgDQpldmlkZW5jZSBvZiBtb2RlbCBmYWlsdXJlLiBBIG1vZGVsIHdoZXJlIGEgZGlhZ25vc3RpYyBwLXZhbHVlIA0KaXMgYmVsb3cgYSBjcml0aWNhbCB2YWx1ZSBpcyBhIG1vZGVsIHRoYXQgKmxpa2VseSogaGFzIGEgcHJvYmxlbSAodGhvdWdoIHdlIA0KY2FuJ3QgdGVsbCB0aGUgKmRlZ3JlZSogb2YgdGhlIHByb2JsZW0gZnJvbSB0aGUgZGlhZ25vc3RpYyBhbG9uZSksIHdoaWxlIGEgDQptb2RlbCB3aXRob3V0IGEgbG93IHAtdmFsdWUgbWF5IG9yIG1heSBub3QuIFRodXMsIHRoaXMgYXBwcm9hY2ggaXMgYSBnb29kIA0KZmlyc3QtcGFzcywgYSBub2lzeSBidXQgdXNlZnVsIGluZGljYXRvciBvZiBob3cgd2VsbCBzcGVjaWZpZWQgdGhlc2UgbW9kZWxzIA0KbWF5IG9yIG1heSBub3QgYmUuIA0KDQpXZSd2ZSB0YWxrZWQgYnJpZWZseSBhYm91dCBzb21lIG9mIHRoZSB0ZXN0cyBpbiB0aGUgbGVjdHVyZSwgYnV0IGhlcmUgd2UnbGwgDQpyZXZpZXcgdGhlbSBpbiBsaWdodCBvZiB0aGUgb3B0aW9ucyB0aGF0IGNhbiBiZSBwYXNzZWQgdG8gdGhlbS4gV2UnbGwganVzdCANCmZvY3VzIG9uIHRlc3RpbmcgYSBzaW5nbGUgbW9kZWwgZm9yIG5vdywgYnV0IGF0IHRoZSBib3R0b20gb2YgdGhpcyB0dXRvcmlhbCANCnRoZXJlIGlzIGNvZGUgZm9yIGNvbWJpbmluZyB0aGUgdGVzdHMgYW5kIHRlc3RpbmcgYWxsIHRoZSBtb2RlbHMgdG9nZXRoZXIuIEZvciANCm5vdywgbGV0J3MgZm9jdXMgb24gaG93IHRoZSB0ZXN0aW5nIGZ1bmN0aW9ucyB3b3JrIGluIFIuIA0KDQojIyMgSGV0ZXJvc2tlZGFzdGljaXR5IFRlc3RzDQoNCi0gQnJldXNjaC1QYWdhbiBUZXN0LCBHb2xmZWxkLVF1YW5kdCBUZXN0DQoNClRoZSBCcmV1c2NoLVBhZ2FuIHRlc3QgZml0cyBhIGxpbmVhciBtb2RlbCB0byB0aGUgcmVzaWR1YWxzIG9mIHRoZSByZWdyZXNzaW9uIA0KbW9kZWwgdXNpbmcgdGhlIHNhbWUgcHJlZGljdG9ycy4gSWYgdG9vIG11Y2ggdmFyaWFuY2UgaXMgZXhwbGFpbmVkIGJ5IHRoZSANCnByZWRpY3RvcnMgaW4gdGhlIG9yaWdpbmFsIG1vZGVsLCB0aGVuIHRoZSBtb2RlbCBpcyBjb25zaWRlcmVkIHRvIGJlIA0KaGV0ZXJvc2tlZGFzdGljLiANCg0KYGBge3J9DQpsaWJyYXJ5KGxtdGVzdCkgIyBsb2FkIHRoZSBsaWJyYXJ5IGZvciBtb2RlbCB0ZXN0aW5nDQpicHRlc3QobTEpICMgQnJldXNjaC1QYWdhbiB0ZXN0LCB0YWtlcyBhIG1vZGVsIG9iamVjdCBhcyBpdHMgYXJndW1lbnQNCmBgYA0KDQpUaGlzIG1vZGVsIGZhaWxzIHRoZSBCcmV1c2NoLVBhZ2FuIHRlc3QgLSB0aGVyZSBpcyBldmlkZW5jZSBvZiBoZXRlcm9za2VkYWN0aWNpdHkgDQpnaXZlbiB0aGUgbW9kZWwgYW5kIHRoZSBkYXRhLiANCg0KVGhlIEdvbGRmZWxkLVF1YW5kdCB0ZXN0IGZpdHMgdGhlIG9yaWdpbmFsIG1vZGVsIHRvIHR3byBzdWJzZXRzIG9mIHRoZSBkYXRhIGRlZmluZWQgDQpieSBhIGJyZWFrcG9pbnQgYW5kIGNvbXBhcmVzIHRoZSB2YXJpYW5jZXMgb2YgdGhlIHJlc2lkdWFscy4gSWYgdGhlIGRpZmZlcmVuY2VzIA0KYXJlIHN0YXRpc3RpY2FsbHkgZGlmZmVyZW50LCB0aGVuIHRoZSBtb2RlbCBpcyBjb25zaWRlcmVkIGhldGVyb3NrZWRhc3RpYy4gQnkgDQpkZWZhdWx0IHRoZSBzYW1wbGUgaXMgc3BsaXQgaW4gaGFsZiwgYnV0IGJlY2F1c2UgaXQgaW52b2x2ZXMgc3BsaXR0aW5nIHRoZSBkYXRhIA0KaXQgaXMgc2Vuc2l0aXZlIHRvIHRoZSBvcmRlciBvZiBvYnNlcnZhdGlvbnMsIGFuZCB3ZSBjYW4gc3BlY2lmeSB0aGUgb3JkZXIgDQp3ZSB3aGljaCB0byBwbGFjZSB0aGUgb2JzZXJ2YXRpb25zIGluLiANCg0KYGBge3J9DQpncXRlc3QobTEpICMgbW9kZWwgb2JqZWN0DQpncXRlc3QobTEsIA0KICAgICAgIG9yZGVyLmJ5ID0gfnNzMSwgDQogICAgICAgZGF0YT1tMSRtb2RlbCwgIyBvbmUgdHJpY2sgaW4gUiBpcyBpZiB5b3Ugd2FudCB0byBwdWxsIHRoZSBkYXRhIGEgDQogICAgICAgIyBtb2RlbCB3YXMgZml0IHRvLCBmb3IgYSBsbSBvYmplY3QgeW91IGNhbiB1c2UgbXlfbW9kZWwkbW9kZWwgdG8gYWNjZXNzIGl0DQogICAgICAgZnJhY3Rpb24gPSAwLjQpDQpgYGANCg0KSGVyZSB3ZSBzZWUgdGhlIGNoYWxsZW5nZSBvZiByZWdyZXNzaW9uIGRpYWdub3N0aWNzIC0gdGhlIEJyZXVzaC1QYWdhbiB0ZXN0IA0Kc2FpZCB3ZSBoYWQgaGV0ZXJvc2tlZGFjdGljaXR5LCBhbmQgc28gZG8gb3VyIHJlc2lkdWFsIHBsb3RzLCBidXQgdGhlIEdRIHRlc3QgDQpkb2VzIG5vdCwgZXZlbiB1bmRlciBkaWZmZXJlbnQgcmVvcmRlcmluZ3Mgb2YgdGhlIGRhdGEuIA0KDQpUaGlzIGlsbHVzdHJhdGVzIHR3byBpc3N1ZXMgdG8gYmUgd2FyeSBvZiB3aGVuIHVzaW5nIHN0YXRpc3RpY2FsIGRpYWdub3N0aWNzLiANCkZpcnN0LCBhbGwgZGlhZ25vc3RpY3MvbWVhc3VyZXMgYXJlIGltcGVyZmVjdCBhbmQgdGhleSBzaG91bGQgYmUgd2VpZ2hlZCANCmNhcmVmdWxseSB3aXRoIG90aGVyIGZvcm1zIG9mIGV2aWRlbmNlLiBJbiB0aGlzIGNhc2UsIHdlIGFyZSBhbHJlYWR5IHNrZXB0aWNhbCANCm9mIHRoZXNlIG1vZGVscyBiYXNlZCBvbiBvdXIgYW5hbHlzaXMgb2YgdGhlIHJlc2lkdWFscyBhbmQgb3RoZXIgZmFjdG9ycyBzbyANCmZpbmRpbmcgb3V0IHRoYXQgaXQgbWF5IGV4aGliaXQgaGV0ZXJvc2tlZGFjdGljaXR5IGlzIG5vdCB1bmV4cGVjdGVkLiBTZWNvbmQsIA0Kc3RhdGlzdGljYWwgZGlhZ25vc3RpY3MsIGxpa2UgYWxsIG90aGVyIHN0YXRpc3RpY3MsIGFyZSBzdXNjZXB0aWJsZSB0byByZXNlYXJjaGVycyANCiJwLWhhY2tpbmciIC0gcnVubmluZyBtdWx0aXBsZSB0ZXN0cyB3aXRoIGRpZmZlcmVudCBwYXJhbWV0ZXJzIHVudGlsIGEgc3VpdGFibHkgDQpsb3cgcC12YWx1ZSBpcyBmb3VuZC4gVHJ5IGl0IHdpdGggdGhlIGBncXRlc3QoKWAgZnVuY3Rpb24gLSBpZiB5b3Ugb3JkZXIgdGhlIA0KZGF0YSB0aGUgcmlnaHQgd2F5IGFuZCBzZXQgdGhlIGBmcmFjdGlvbmAgYXJndW1lbnQgY29ycmVjdGx5LCB5b3Ugd2lsbCANCnVuZG91YnRlZGx5IGZpbmQgYSBwLXZhbHVlIGJlbG93IHRoZSBjcml0aWNhbCB2YWx1ZS4gDQoNCiMjIyBBdXRvY29ycmVsYXRpb24gRGlhZ25vc3RpY3MNCg0KLSBEdXJiaW4tV2F0c29uIFRlc3QsIEJveC1QaWVyY2UgVGVzdCwgTGp1bmctQm94IFRlc3QsIEJyZXVzY2gtR29kZnJleSBUZXN0DQoNCkF1dG9jb3JyZWxhdGlvbiBpcyBhIHByb2JsZW0gd2l0aCByZWdyZXNzaW9uIG1vZGVscyB3aGVyZSB0aGUgcmVzaWR1YWxzIG9mIA0Kb25lIG9ic2VydmF0aW9uIGFyZSBjb3JyZWxhdGVkIHdpdGggdGhlIHJlc2lkdWFscyBvZiBvdGhlciBvYnNlcnZhdGlvbnMgaW4gdGhlIA0KZGF0YS4gTW9zdCBjb21tb25seSwgYXV0b2NvcnJlbGF0aW9uIG9jY3VycyB3aGVuIG9ic2VydmF0aW9ucyBhcmUgb3JkZXJlZCANCmJ5IHRpbWUgYW5kIHRoZSBvYnNlcnZhdGlvbiBvZiBhIHVuaXQgaW4gdGltZSB0IGlzIGNvcnJlbGF0ZWQgd2l0aCBpdHMgdmFsdWUgDQphdCB0aW1lIHQtMSAodGhpbmsgdGhlIHRlbXBlcmF0dXJlIGRheSB0byBkYXkgYW55d2hlcmUgaW4gdGhlIHdvcmxkIGV4Y2VwdCANCk1vbnRhbmEpLiANCg0KQXV0b2NvcnJlbGF0aW9uIGRpYWdub3N0aWNzLCB0byBkZXRlY3QgdGhpcywgIHJlbHkgb24gdGhlIG9yZGVyaW5nIG9mIHRoZQ0Kb2JzZXJ2YXRpb25zLiBUaGV5IGFyZSANCmVzcGVjaWFsbHkgaW1wb3J0YW50IHdoZW4gbW9kZWxpbmcgdGltZS1zZXJpZXMgZGF0YSBhbmQgaW5jbHVkaW5nIHRpbWUgYXMgYSANCmZlYXR1cmUgaW4gdGhlIG1vZGVsLiBPdXIgbW9kZWwgaGVyZSBkb2VzIHRoaXMgaW1wbGljaXRseSBtb2RlbGluZyB0ZXN0IHNjb3JlcyANCmluIHRpbWUgdCBhcyBhIGZ1bmN0aW9uIG9mIHRoZSBzY29yZXMgaW4gdGltZSB0LTEgYW5kIHdlIGhhdmUgYSB0ZXN0IHllYXIgDQp2YXJpYWJsZSBpbiB0aGUgZGF0YS4gDQoNCg0KDQpgYGB7cn0NCmR3dGVzdChzczIgfiBzczEsIGRhdGEgPSBieV9ncm91cCRkYXRhW1s2XV0pDQpCb3gudGVzdChyZXNpZHVhbHMobG0oc3MyfnNzMSwgZGF0YSA9IGJ5X2dyb3VwJGRhdGFbWzZdXSkpKQ0KQm94LnRlc3QocmVzaWR1YWxzKGxtKHNzMn5zczEsIGRhdGEgPSBieV9ncm91cCRkYXRhW1s2XV0pKSwgdHlwZSA9ICJManVuZyIpDQpiZ3Rlc3Qoc3MyIH4gc3MxLCBkYXRhID0gYnlfZ3JvdXAkZGF0YVtbNl1dKQ0KYGBgDQoNClNpbmNlIHdlIGhhdmUgYWxyZWFkeSBzcGxpdCBvdXIgZGF0YSBvdXQgYnkgdGVzdCB5ZWFyLCB3ZSBoYXZlIGJyb2tlbiB1cCB0aGUgDQptb3N0IG9idmlvdXMgc291cmNlIG9mIGF1dG9jb3JyZWxhdGlvbiBpbiBvdXIgZ3JvdXAtd2lzZSByZWdyZXNzaW9uIG1vZGVsIA0Kc3RyYXRlZ3kuIExldCdzIGxvb2sgYXQgc29tZSBvdGhlciBwb3RlbnRpYWwgd2F5cyB3ZSBjb3VsZCBpbWFnaW5lIGF1dG9jb3JyZWxhdGlvbiANCm9jY3VycmluZy4gDQoNCkhvd2V2ZXIsIG1vcmUgZ2VuZXJhbGx5LCBhdXRvY29ycmVsYXRpb24gY2FuIG9jY3VyIGlmIHRoZXJlIGlzIHNvbWUgZ3JvdXBpbmcgb2YgDQpvYnNlcnZhdGlvbnMgdGhhdCBpc24ndCBhY2NvdW50ZWQgZm9yIC0tIGNvbW1vbiBpbiBlZHVjYXRpb24uIEZvciBleGFtcGxlLCBpbiANCm91ciBkYXRhLCB0aGUgZ3JhZGUtc3ViamVjdC10ZXN0X3llYXIgc2Nob29sIGF2ZXJhZ2Ugb2JzZXJ2YXRpb25zIGFyZSBuZXN0ZWQgDQp3aXRoaW4gZGlzdHJpY3RzIGFuZCBzY2hvb2xzIHdpdGhpbiB0aGUgc2FtZSBkaXN0cmljdCBhcmUgbW9yZSBzaW1pbGFyIHRvIG9uZSANCmFub3RoZXIgdGhhbiBzY2hvb2xzIGluIGRpZmZlcmVudCBkaXN0cmljdHMuIEZhaWxpbmcgdG8gYWNjb3VudCBmb3IgdGhpcyBjb3VsZCANCmNhdXNlIGF1dG9jb3JyZWxhdGlvbiBpbiBvdXIgbW9kZWxzLiANCg0KDQpgYGB7cn0NCmR3dGVzdChzczIgfiBzczEsIGRhdGEgPSBieV9ncm91cCRkYXRhW1s2XV0sIG9yZGVyLmJ5ID0gfmRpc3RpZCkNCmJndGVzdChzczIgfiBzczEsIGRhdGEgPSBieV9ncm91cCRkYXRhW1s2XV0sIG9yZGVyLmJ5ID0gfmRpc3RpZCkNCg0KYGBgDQoNCldoZW4gd2UgbG9vayBhdCBpdCBub3QgdGVtcG9yYWxseSwgYnV0IG9yZ2FuaXphdGlvbmFsbHksIHdlIHNlZSBldmlkZW5jZSB0aGF0IA0KdGhlcmUgaXMgaW5kZWVkIGF1dG9jb3JyZWxhdGlvbiBpbiBvdXIgZGF0YSAtLSBvYnNlcnZhdGlvbnMgZnJvbSB0aGUgc2FtZSANCmRpc3RyaWN0IGFyZSBjb3JyZWxhdGVkIGVub3VnaCB3aXRoIG9uZSBhbm90aGVyIHRvIGNhdXNlIHByb2JsZW1zLiANCg0KIyMjIE11bHRpY29sbGluZWFyaXR5DQoNCkFzaWRlIGZyb20gYmVpbmcgdGhlIG1vc3Qgc3RyYWlnaHRmb3J3YXJkIGNyaXRpcXVlIG9mIGFueSBzb2NpYWwgc2NpZW5jZSBzdGF0aXN0aWNhbA0KbW9kZWwsIG11bHRpY29sbGluZWFyaXR5IGlzIGEgYmlnIHByb2JsZW0gZm9yIGludGVycHJldGluZyBtYWduaXR1ZGUgYW5kDQpzdGF0aXN0aWNhbCBzaWduaWZpY2FuY2Ugb2YgaW5kaXZpZHVhbCBwcmVkaWN0b3JzIGluIGEgbW9kZWwuIElmIHR3byBoaWdobHkNCmNvcnJlbGF0ZWQgdmFyaWFibGVzIGFyZSBpbmNsdWRlZCBpbiB0aGUgbW9kZWwsIHRoZW4gdGhlaXIgY29lZmZpY2llbnRzIHdpbGwgYmUNCmEgYmxlbmQgb2YgdGhlIGVmZmVjdCBvZiBib3RoIHZhcmlhYmxlcyBhbmQgdGhlIHN0YW5kYXJkIGVycm9ycyBmb3IgdGhlc2UNCnZhcmlhYmxlcyB3aWxsIGJlIGluZmxhdGVkIGFuZCB1bnJlbGlhYmxlLg0KDQpGcm9tIFVDTEEncyBTdGF0YSBSZWdyZXNzaW9uIGRpYWdub3N0aWMgZ3VpZGU6DQo+IEFzIGEgcnVsZSBvZiB0aHVtYiwgYSB2YXJpYWJsZSB3aG9zZSBWSUYgdmFsdWVzIGFyZSBncmVhdGVyIHRoYW4gMTAgbWF5IG1lcml0IGZ1cnRoZXIgaW52ZXN0aWdhdGlvbi4gSXQgIG1lYW5zIHRoYXQgdGhlIHZhcmlhYmxlIGNvdWxkIGJlIGNvbnNpZGVyZWQgYXMgYSBsaW5lYXIgY29tYmluYXRpb24gb2Ygb3RoZXIgaW5kZXBlbmRlbnQgdmFyaWFibGVzLg0KDQpgYGB7cn0NCiMgaW5zdGFsbC5wYWNrYWdlcygiY2FyIikgaWYgeW91IGRvbid0IGFscmVhZHkgaGF2ZSB0aGUgY2FyIHBhY2thZ2UNCg0KIyBGaXQgYSBtb3JlIGNvbXBsZXggbW9kZWwgYXMgYW4gZXhhbXBsZQ0KdG1wX2ZpdCA8LSBsbShzczIgfiBzczEgKyBmYWN0b3IoZ3JhZGUpICogZmFjdG9yKHRlc3RfeWVhcikgKyBmYWN0b3Ioc3ViamVjdCkgKyBuMSwgZGF0YSA9IGZ1bGxfZGF0YSkNCmNhcjo6dmlmKHRtcF9maXQpDQoNCmBgYA0KDQoNCiMjIyBNaXNzcGVjaWZpY2F0aW9uIERpYWdub3N0aWNzDQoNCi0gUmFpbmJvdyBUZXN0LCBSRVNFVCBUZXN0LCBIYXJ2ZXktQ29sbGllciBUZXN0DQoNCk9uZSBmb3JtIG9mIG9tbWl0dGVkIHZhcmlhYmxlIGJpYXNlIHdlIGNhbiBlYXNpbHkgdGVzdCBmb3IgaXMgd2hldGhlciBjb250aW51b3VzDQptZWFzdXJlcyBpbiBvdXIgbW9kZWwgd291bGQgYmUgYmV0dGVyIGRlc2NyaWJlZCBieSBwb2x5bm9taWFsIHRlcm1zIGluc3RlYWQgb2YgYQ0Kc2luZ2xlIGxpbmVhciBjb2VmZmljaWVudC4gVGVzdHMgb2YgdGhpcyB1c2UgZGlmZmVyZW50IHN0cmF0ZWdpZXMuDQoNClRoZSBSYWluYm93IHRlc3QgZml0cyBhIHJlZ3Jlc3Npb24gdG8gYSBtaWRkbGUgcG9ydGlvbiBvZiB0aGUgZGF0YSBhbmQgZG9lcyANCmEgdGVzdCB0byBzZWUgaWYgaXQgZml0cyB0aGUgZW5kcyBvZiB0aGUgZGF0YSBhcyB3ZWxsIGFzIHRoZSBtaWRkbGUuIElmIHRoZSANCmZpdCBmb3IgZWl0aGVyIHRhaWwgaXMgd29yc2UgdGhhbiBmb3IgdGhlIGNlbnRlciBzdWJzYW1wbGUsIHRoaXMgaXMgdGFrZW4gYXMgDQpldmlkZW5jZSB0aGUgbW9kZWwgY291bGQgYmUgaW1wcm92ZWQgYnkgcG9seW5vbWlhbCB0ZXJtcy4gDQoNCmBgYHtyfQ0KcmFpbnRlc3Qoc3MyIH4gc3MxLCBkYXRhID0gYnlfZ3JvdXAkZGF0YVtbNl1dKQ0KYGBgDQoNClRoZSBSRVNFVCB0ZXN0IGZpdHMgYSBtb2RlbCB3aXRoIGEgdXNlci1zcGVjaWZpZWQgbnVtYmVyIG9mIGFkZGl0aW9uYWwgcG93ZXJzIA0Kb2YgdGhlIFggdmFyaWFibGVzIGFuZCB0aGVuIGRvZXMgYSBzdGFuZGFyZCBGLXRlc3QgdG8gc2VlIGlmIHRoaXMgbmV3IG1vZGVsIA0Kb3V0cGVyZm9ybXMgdGhlIG9yaWdpbmFsIG1vZGVsLiBUaGUgbW9zdCBjb21tb24gdGVzdCBpcyB0aGUgUmFtc2V5IFJlZ3Jlc3Npb24gRXF1YXRpb24gU3BlY2lmaWNhdGlvbiBFcnJvciBUZXN0LCANCm9yIFJFU0VULCB3aGljaCBjaGVja3Mgd2hldGhlciB0aGVyZSBhcmUgb21pdHRlZCBwb2x5bm9taWFsIHRlcm1zIHRvIGEgcHJlZGljdG9yIA0KdGhhdCB3b3VsZCBpbXByb3ZlIG1vZGVsIGZpdC4gDQoNClRoaXMgdGVsbHMgdXMgdGhhdCB0aGUgc3VtIG9mIHNxdWFyZXMgd2lsbCBiZSBtZWFuaW5nZnVsbHkgcmVkdWNlZCB3aXRoIHRoZSANCnNpbXBsZSBpbmNsdXNpb24gb2YgcG9seW5vbWlhbHMgb2YgdGhlIG1haW4gcHJlZGljdG9yIC0tIGBzczFgLiBUaHVzIHRoZSANCm5ldyBtb2RlbCBzaG91bGQgdGFrZSB0aGUgZm9ybToNCg0KJCRTU197Mn0gPSBcYWxwaGEgKyBcYmV0YV97MX1TU197MX0gKyAgXGJldGFfezJ9U1NfezF9XjIgKyBcYmV0YV97M31TU197MX1eMyArIFxiZXRhX3s0fVNTX3sxfV40JCQNCg0KDQpgYGB7cn0NCnJlc2V0KHNzMiB+IHNzMSwgcG93ZXI9IDI6NCwgZGF0YSA9IGJ5X2dyb3VwJGRhdGFbWzZdXSkNCmBgYA0KDQoNClRoZSBIYXJ2ZXktQ29sbGllciB0ZXN0IGNvbmR1Y3RzIGEgdC10ZXN0IG9uIHRoZSBtZWFucyBvZiB0aGUgcmVjdXJzaXZlIHJlc2lkdWFscy4gDQpUaGUgbWVhbnMgc2hvdWxkIG5vdCBkaWZmZXIgZnJvbSAwIHNpZ25pZmljYW50bHksIHNvIGlmIHRoZXkgZG8sIHRoaXMgaXMgdGFrZW4gDQphcyBldmlkZW5jZSBzdXBwb3J0aW5nIHRoZSBuZWVkIGZvciBwb2x5bm9taWFsIHRlcm1zLiANCg0KDQpgYGB7cn0NCmhhcnZ0ZXN0KHNzMiB+IHNzMSwgZGF0YSA9IGJ5X2dyb3VwJGRhdGFbWzZdXSkNCmBgYA0KDQpUaGVzZSBjYW4gYWxzbyANCm9wdGlvbmFsbHkgYmUgb3JkZXJlZCBieSBhIHZlY3RvciB0aGF0IG1heSBiZSBtZWFuaW5nZnVsIHN1Y2ggYXMgYSBncm91cGluZyANCnRlcm0gb3IgdGhlIHZhbHVlIG9mIGEgcHJlZGljdG9yLiBXZSBtaWdodCBjb25zaWRlciBvcmRlcmluZyB0aGUgZGF0YSBieSANCmNsYXNzIHNpemUgKGBuMWApIG9yIHByZS10ZXN0IHNjb3JlIChgc3MxYCkgb3Igc2Nob29sIGRpc3RyaWN0IGBkaXN0aWRgLiANCg0KYGBge3J9DQpyYWludGVzdChzczIgfiBzczEsIGRhdGEgPSBieV9ncm91cCRkYXRhW1s2XV0sIG9yZGVyLmJ5ID0gfnNzMSkNCmhhcnZ0ZXN0KHNzMiB+IHNzMSwgZGF0YSA9IGJ5X2dyb3VwJGRhdGFbWzZdXSwgb3JkZXIuYnkgPSB+c3MxKQ0KYGBgDQoNClRoZSBzYW1lIGNvbmNlcm5zIGFib3V0IHAtaGFja2luZyBhcHBseS4gDQoNCiMjIE1vZGVsIENvbXBhcmlzb25zDQoNCkFmdGVyIHdlIGhhdmUgZGlhZ25vc2VkIHRoZSBtb2RlbCwgd2Ugd2FudCB0byBwcm9wb3NlIGFsdGVybmF0aXZlIG1vZGVscywgYW5kIA0Kd2UgbmVlZCBhIGZyYW1ld29yayBmb3IgYXNzZXNzaW5nIHRoZSBkaWZmZXJlbmNlIGJldHdlZW4gYW55IGFsdGVybmF0aXZlcyBhbmQgDQpvdXIgZXhpc3RpbmcgbW9kZWwuIA0KDQpSZW1lbWJlciB0aGF0IGluIG91ciBwcmV2aW91cyB3b3JrIHdlIG5vdCBvbmx5IGlkZW50aWZpZWQgYWx0ZXJuYXRpdmUgbW9kZWxzIA0KdXNpbmcgdGhlIGFkZGl0aW9uYWwgZGF0YSB3ZSd2ZSBtZXJnZWQgaW4sIGJ1dCB3ZSBldmVuIGZvdW5kIGltcHJvdmVkIG1vZGVsIA0KZml0IGJ5IGluY2x1ZGluZyBwb2x5bm9taWFsIHJlZ3Jlc3Npb24gdGVybXMgdG8gYWNjb3VudCBmb3IgdGhlIG5vbi1saW5lYXIgDQpyZWxhdGlvbnNoaXAgd2Ugb2JzZXJ2ZWQgYmV0d2VlbiBgc3MxYCBhbmQgYHNzMmAgYXQgdGhlIHRhaWxzIG9mIHRoZSBkaXN0cmlidXRpb24uIA0KDQpUaGVyZSBhcmUgbXVsdGlwbGUgd2F5cyB0byBjb21wYXJlIG1vZGVscyBkZXBlbmRpbmcgb24gdGhlIHB1cnBvc2VzIG9mIHRoZSBtb2RlbCANCmFuZCB0aGUgZ29hbHMgb2YgdGhlIGFuYWx5c3QuIFdlIHdpbGwgZGlzY3VzcyB0aHJlZSBoZXJlOiANCg0KMS4gQ29tcGFyaW5nIG1vZGVsIGZpdCB0byB0aGUgZGF0YQ0KMi4gQ29tcGFyaW5nIG1vZGVsIGltcHJvdmVtZW50IHdpdGggRi10ZXN0cw0KMy4gQ29tcGFyaW5nIG1vZGVsIHByZWRpY3RpdmUgcG93ZXINCg0KSW4gdGhpcyBzZWN0aW9uLCB3ZSdsbCBsb29rIGF0IGhvdyB3ZSBjYW4gZG8gbW9kZWwgY29tcGFyaXNvbiBhY3Jvc3Mgb3VyIGZpZnR5IA0KbW9kZWxzLiBUaGlzIHdpbGwgYWxsb3cgdXMgdG8gY29tcGFyZSB0aGUgc2FtZSBtb2RlbCBhY3Jvc3MgZGlmZmVyZW50IHN1YnNldHMgDQpvZiBkYXRhICh3aXRoaW4gdGhlIGZpZnR5IG1vZGVscykgYW5kIGNvbXBhcmUgbmV3IG1vZGVsIGZpdHMgdG8gdGhlIHNhbWUgZGF0YSANCihjb21wYXJlIG1vZGVsIEEgdG8gbW9kZWwgQiBhY3Jvc3MgZWFjaCBvZiB0aGUgZmlmdHkgc2V0cyBvZiBkYXRhKS4gQm90aCANCm9mIHRoZXNlIHN0cmF0ZWdpZXMgYXJlIGhlbHBmdWwgd2hlbiBldmFsdWF0aW5nIG1vZGVscyByZXBlYXRlZCBhY3Jvc3Mgc3Vic2V0cy4gDQoNCkZpcnN0LCBsZXQncyBsb29rIGF0IGEgbWV0aG9kIHdlIGNhbiB1c2UgdG8gZ2V0IGFuZCBwbG90IHRoZSBtb2RlbCBmaXRzIGZvciBlYWNoIA0Kb2Ygb3VyIDUwIHN1Ym1vZGVscyB1c2luZyB0aGUgYGdsYW5jZWAgZnVuY3Rpb24gdG8gZXh0cmFjdCB0aGUgbW9kZWwgZml0IHN0YXRpc3RpY3MgDQpmcm9tIGVhY2ggZ3JvdXAuIA0KDQpgYGB7cn0NCmdsYW5jZSA8LSBieV9ncm91cCAlPiUNCiAgbXV0YXRlKGdsYW5jZSA9IG1hcChtb2RlbCwgYnJvb206OmdsYW5jZSkpICU+JQ0KICB1bm5lc3QoZ2xhbmNlLCAuZHJvcCA9IFRSVUUpDQoNCmFycmFuZ2UoZ2xhbmNlLCBkZXNjKHIuc3F1YXJlZCkpDQoNCg0KZ2dwbG90KGdsYW5jZSwgYWVzKHggPSBncmFkZSwgeSA9IHIuc3F1YXJlZCwgY29sb3IgPSBzdWJqZWN0KSkgKw0KICBnZW9tX2ppdHRlcih3aWR0aCA9IDAuMSwgaGVpZ2h0ID0gMCkgKyB0aGVtZV9idygpDQoNCmBgYA0KDQpUaGlzIHBsb3Qgc2hvd3MgNTAgcG9pbnRzLCBvbmUgZm9yIGVhY2ggY29tYmluYXRpb24gb2YgZ3JhZGUsIHllYXIsIGFuZCBzdWJqZWN0LiANCkVhY2ggcG9pbnQgcmVwcmVzZW50cyB0aGUgci1zcXVhcmVkIG9mIG91ciBwcm9wb3NlZCBtb2RlbCAoc3MyIH4gc3MxKSBmaXQgdG8gDQp0aGF0IHNwZWNpZmljIHN1YnNldCBvZiB0aGUgZGF0YS4gVGhlIHBsb3Qgc2hvd3MgYSBjbGVhciBwYXR0ZXJuLCBvdXIgbW9kZWwgDQpmaXRzIGJldHRlciBpbiBoaWdoZXIgZ3JhZGVzIHRoYW4gaW4gbG93ZXIgZ3JhZGVzLiANCg0KQmVmb3JlIHdlIGdvIGNvbXBhcmluZyBhbHRlcm5hdGl2ZXMgdG8gYWxsIGZpZnR5IG1vZGVscywgbGV0J3MgbG9vayBhdCB0aGUgdGhyZWUNCndheXMgd2UgY2FuIGNvbXBhcmUgbW9kZWxzIHRocm91Z2ggdGhlIGxlbnMgb2YgYSBhIHNpbmdsZSB5ZWFyLWdyYWRlLXN1YmplY3QNCm1vZGVsLiBXZSdsbCBzdGFydCBieSBhZGRpbmcgdGhlIGNsYXNzIHNpemUsIG9uZSBvZiB0aGUgb3B0aW9ucyB3ZSBleHBsb3JlZCBpbg0KdGhlIHByZXZpb3VzIGd1aWRlZCB0dXRvcmlhbCwgdG8gYnVpbGQgYW4gYWx0ZXJuYXRpdmU6DQoNCmBgYHtyfQ0KbTEgPC0gYnlfZ3JvdXAkbW9kZWxbWzE1XV0NCm0yIDwtIGxtKHNzMiB+IHNzMSArIG4xLCBkYXRhID0gYnlfZ3JvdXAkZGF0YVtbMTVdXSkNCnN1bW1hcnkobTEpDQpzdW1tYXJ5KG0yKQ0KYGBgDQoNCkxldCdzIGxvb2sgYXQgdGhlICJNdWx0aXBsZSBSLXNxdWFyZWQiIG1lYXN1cmUgaGVyZS4gRm9yIHRoZSBmaXJzdCBtb2RlbCwgaXQgDQppcyBgciBtb2RlbHI6OnJzcXVhcmUobTEsIGJ5X2dyb3VwJGRhdGFbWzE1XV0pYCBhbmQgZm9yIHRoZSBzZWNvbmQgbW9kZWwgaXQgaXMgDQpgciBtb2RlbHI6OnJzcXVhcmUobTIsIGJ5X2dyb3VwJGRhdGFbWzE1XV0pYC4gVGhpcyBpcyBhIGRpZmZlcmVuY2Ugb2YgMC4wMDUuIFdoaWxlIHRoZXJlIA0KYXJlIG5vIGhhcmQgZGVjaXNpb24gcnVsZXMgZm9yIGhvdyBtdWNoIGltcHJvdmVtZW50IGluIHItc3F1YXJlZCBpcyBtZWFuaW5nZnVsIA0Kb3IgYSBzaWduaWZpY2FudCBpbXByb3ZlbWVudCAobW9yZSBvbiB0aGlzIGxhdGVyKSwgd2UgY2FuIGJlIGNvbmZpZGVudCB0aGF0IHRoaXMgDQppcyBub3QgaXQuIA0KDQpIb3dldmVyLCBpdCBjYW4gYmUgaGVscGZ1bCB0byBmb3JtYWxseSBtZWFzdXJlIHRoZXNlIHRoaW5ncy4gV2UgY2FuIHVzZSBhIA0KZm9ybWFsIEYtdGVzdCB0byBkaXJlY3RseSBjb21wYXJlIHRoZXNlIHR3byBtb2RlbHMgc2luY2UgdGhlIA0Kb3JpZ2luYWwgbW9kZWwgaXMgZnVsbHkgbmVzdGVkIHdpdGhpbiB0aGUgbmV3IG1vZGVsIC0tIHRoZXkgYXJlIGlkZW50aWNhbCBleGNlcHQgDQpmb3IgdGhlIGFkZGl0aW9uIG9mIG9uZSBwcmVkaWN0b3IgYW5kIGZpdCB0byB0aGUgc2FtZSBzYW1wbGUuIEFuIEYtdGVzdCBnaXZlcyANCnVzIGEgcC12YWx1ZSB0byB0ZWxsIHVzIHdoZXRoZXIgdGhlIGltcHJvdmVtZW50IGluIHRoZSBmaXQgdG8gdGhlIGRhdGEgYmV0d2VlbiANCnRoZSB0d28gbW9kZWxzIHdhcyBhIHN0YXRpc3RpY2FsbHkgc2lnbmlmaWNhbnQgZGlmZmVyZW5jZSBvciBub3QuIA0KDQpDb25mdXNpbmdseSwgdGhlIFIgY29tbWFuZCB0byBjb25kdWN0IHRoaXMgRi10ZXN0IGlzIGNhbGxlZCBgYW5vdmEoKWANCg0KYGBge3J9DQphbm92YShtMSwgbTIpDQpgYGANCg0KU28gZXZlbiB3aXRoIGEgdmVyeSBtb2Rlc3QgaW1wcm92ZW1lbnQgaW4gUi1zcXVhcmVkLCB0aGUgRi10ZXN0IHRlbGxzIHVzIHRoYXQgDQp0aGUgZGF0YSBzdXBwb3J0IHRoZSBjb25jbHVzaW9uIHRoYXQgbW9kZWwgMiBpcyBhIGJldHRlciBmaXQgdG8gdGhlIGRhdGEgdGhhbiANCm1vZGVsIDEuIFRoaXMgaXMgYSBjYXNlIHdoZXJlIHdlIG5lZWQgdG8gY29uc2lkZXIgb3VyIHF1ZXN0aW9uIGFuZCBvdXIgZ29hbHMgDQp0byBkZXRlcm1pbmUgd2hldGhlciB0aGUgbW9kZWwgaXMgc3VwZXJpb3Igb3Igbm90LiANCg0KRi10ZXN0cyBhcmUgcmVhbGx5IHVzZWZ1bCBiZWNhdXNlIHRoZXkgYWxsb3cgdXMgdG8gdGVzdCB0aGUgam9pbnQgc2lnbmlmaWNhbmNlIA0Kb2YgbXVsdGlwbGUgdmFyaWFibGVzLiBXZSBjYW4gYWxzbyB0ZXN0IG11bHRpcGxlIG5lc3RlZCBtb2RlbHMgYXQgb25jZS4gSGVyZSwgDQpsZXQncyBhZGQgYSBzZXQgb2YgcG9seW5vbWlhbCB0ZXJtcyBhbmQgam9pbnRseSB0ZXN0IHdoZXRoZXIgYXMgYSBncm91cCB0aGV5IA0KcHJvdmlkZSBhIHN0YXRpc3RpY2FsbHkgc2lnbmlmaWNhbnQgYmV0dGVyIG1vZGVsIGZpdC4gDQoNCmBgYHtyfQ0KbTMgPC0gbG0oc3MyIH4gc3MxICsgSShzczFeMikgKyBJKHNzMV4zKSArIEkoc3MxXjQpICsgbjEsIA0KICAgICAgICAgZGF0YSA9IGJ5X2dyb3VwJGRhdGFbWzE1XV0pDQphbm92YShtMSwgbTIsIG0zKQ0KIyBzdW1tYXJ5KG0zKQ0KYGBgDQoNCkhlcmUsIHdlIHNlZSB0aGF0IHRoZSBwb2x5bm9taWFsIHRlcm1zIGFnYWluIHByb3ZpZGUgYSBzdGF0aXN0aWNhbGx5IHNpZ25pZmljYW50IA0KaW1wcm92ZW1lbnQgdG8gZml0LCBldmVuIG92ZXIgb3VyIG1vZGVsIHdpdGggY29ob3J0IHNpemUuIA0KDQpgYGB7cn0NCnN1bW1hcnkobTMpDQpgYGANCg0KV2Ugc2VlIHRoYXQgb3VyIFItc3F1YXJlZCBoYXMgaW5jcmVhc2VkIGFuZCBlYWNoIGluZGl2aWR1YWwgdmFyaWFibGUgaXMgDQpzdGF0aXN0aWNhbGx5IHNpZ25pZmljYW50LCBzbyB0aGUgRi10ZXN0IG1ha2VzIHNvbWUgc2Vuc2UuIFVzaW5nIEYtdGVzdHMgaXMgYSANCnVzZWZ1bCB3YXkgdG8gdW5kZXJzdGFuZCB0aGUgaW1wYWN0IGFkZGl0aW9uYWwgcHJlZGljdG9ycyBoYXZlIG9uIG91ciBtb2RlbCwgc28gDQppdCdzIGEgZ29vZCB0b29sIHRvIGhhdmUgaW4geW91ciB0b29sYmVsdC4gDQoNCkxldCdzIGxvb2sgYXQgaG93IHdlIGNhbiBhcHBseSB0aGlzIEYtdGVzdCBhcHByb2FjaCB0byBtb2RlbHMgYWNyb3NzIGRpZmZlcmVudCANCnN1YnNldHMgaW4gb3VyIGV4YW1wbGUgcHJvYmxlbSBoZXJlLiANCg0KDQpgYGB7cn0NCiMgRGVmaW5lIGEgZ3JvdXBlZCBkYXRhZnJhbWUgdXNpbmcgb25seSB0aGUgY29sdW1ucyB3ZSBuZWVkICgxOjEwKQ0KIyBncm91cCBieSB0aGUgZ3JhZGUsIHN1YmplY3QsIGFuZCB5ZWFyDQojIFRoZW4gbmVzdCBzbyB0aGUgZGF0YSBoYW5ncyB0b2dldGhlciBpbiBvbmUgbGlzdA0KYnlfZ3JvdXAgPC0gZnVsbF9kYXRhWywgMToxMF0gJT4lDQogIGdyb3VwX2J5KGdyYWRlLCBzdWJqZWN0LCB0ZXN0X3llYXIpICU+JQ0KICBuZXN0KCkNCg0KIyBEZWZpbmUgYSBzaW1wbGUgZnVuY3Rpb24gdGhhdCBmaXRzIG91ciBtb2RlbA0Kc2ltcGxlX21vZGVsIDwtIGZ1bmN0aW9uKGRmKSB7DQogIGxtKHNzMiB+IHNzMSwgZGF0YSA9IGRmKQ0KfQ0KDQphbHRfbW9kZWwgPC0gZnVuY3Rpb24oZGYpIHsNCiAgbG0oc3MyIH4gc3MxICsgSShzczFeMikgKyBJKHNzMV4zKSArIEkoc3MxXjQpICsgbjEsIGRhdGEgPSBkZikNCn0NCg0KIyBBcHBseSB0aGF0IG1vZGVsIHRvIGVhY2ggZ3JvdXAgaW4gb3VyIGRhdGFzZXQgdXNpbmcgdGhlIGBtYXBgIGZ1bmN0aW9uDQojIFRoaXMgaXMgZXF1aXZhbGVudCB0byBhIGxvb3AsIGJ1dCBtb3JlIGVmZmljaWVudCB0byB3cml0ZQ0KYnlfZ3JvdXAgPC0gYnlfZ3JvdXAgJT4lDQogIG11dGF0ZShiYXNlX21vZGVsID0gbWFwKGRhdGEsIHNpbXBsZV9tb2RlbCksIA0KICAgICAgICAgZnVsbF9tb2RlbCA9IG1hcChkYXRhLCBhbHRfbW9kZWwpKQ0KDQojIEFwcGx5IGEgZnVuY3Rpb24gdGhhdCBtYWtlcyBhbiBGLXRlc3Qgb2Ygb3VyIHR3byBtb2RlbHMgKHggYW5kIHkpDQojIHJldHVybnMgb25seSB0aGUgc3RhdGlzdGljcyBmcm9tIHRoZSBhbm92YSBvYmplY3Qgd2Ugd2FudA0KIyBpbiB0aGlzIGNhc2UsIHRoZSBkaWZmZXJlbmNlIGluIHRoZSBzdW0gb2Ygc3F1YXJlcyBhbmQgdGhlIHAtdmFsdWUgb2YgdGhlIA0KIyBGLXRlc3QNCnNpbXBsZV9hbm92YSA8LSBmdW5jdGlvbih4LCB5LCBzdGF0ID0gYygic3MiLCAicCIpLCAuLi4pew0KICBvdXQgPC0gYW5vdmEoeCwgeSkNCiAgaWYoc3RhdCA9PSAic3MiKXsNCiAgICByZXR1cm4ob3V0JGBTdW0gb2YgU3FgW1syXV0pDQogIH0gZWxzZSBpZihzdGF0ID09ICJwIil7DQogICAgcmV0dXJuKG91dCRgUHIoPkYpYFtbMl1dKQ0KICB9DQp9DQoNCmZ0ZXN0cyA8LSBieV9ncm91cCAlPiUgcm93d2lzZSgpICU+JQ0KICAgIGRvKHNzID0gdW5saXN0KHNpbXBsZV9hbm92YSh4ID0gLiRiYXNlX21vZGVsLCB5ID0gLiRmdWxsX21vZGVsLCBzdGF0ID0gInNzIikpLCANCiAgICAgIHB2YWwgPSB1bmxpc3Qoc2ltcGxlX2Fub3ZhKHggPSAuJGJhc2VfbW9kZWwsIHkgPSAuJGZ1bGxfbW9kZWwsIHN0YXQgPSAicCIpKSkNCg0KZnRlc3RzIDwtIGFwcGx5KGZ0ZXN0cywgMiwgdW5saXN0KSAjIGNvbnZlcnQgdG8gYSBudW1lcmljIG9iamVjdA0KZnRlc3RzIDwtIGFzLmRhdGEuZnJhbWUoZnRlc3RzKSAjIG1ha2UgaW50byBhIGRhdGEuZnJhbWUgZm9yIGVhc2Ugb2YgdXNlDQp0YWJsZShmdGVzdHMkcHZhbCA8IDAuMDUpDQpgYGANCg0KVGhpcyB0YWJsZSBpcyB0ZWxsaW5nIHVzIHRoYXQgaW4gcm91Z2hseSBoYWxmIG9mIHRoZSBjYXNlcywgdGhlIGRhdGEgc3VwcG9ydCB0aGUNCmNvbmNsdXNpb24gdGhhdCB0aGUgZnVsbGVyIG1vZGVsIGJldHRlciBkZXNjcmliZXMgdGhlIGRhdGEuIA0KDQpBcyB3ZSBzYXcsIHRoZSByLXNxdWFyZWQgaW1wcm92ZW1lbnQgd2FzIG1hcmdpbmFsLCBidXQgdGhlIEYtdGVzdCBpbXByb3ZlbWVudCANCndhcyBzdGF0aXN0aWNhbGx5IHNpZ25pZmljYW50LiBUaGVzZSBhcmUgYW5zd2VyaW5nIHNsaWdodGx5IGRpZmZlcmVudCBxdWVzdGlvbnMsIA0KYmVjYXVzZSB0aGUgRi10ZXN0IGlzIGludGVuZGVkIHRvIHNwZWNpZmljYWxseSB0YWtlIGludG8gYWNjb3VudCB0aGUgZmFjdCB0aGF0IA0KZXZlcnkgYWRkaXRpb25hbCB2YXJpYWJsZSBhZGRlZCBpbXByb3ZlcyBtb2RlbCBmaXQgdG8gdGhlIGRhdGEsIHNvIHRoZSBpbXByb3ZlbWVudCANCm11c3QgYmUgYmlnIGVub3VnaCB0byBiZSBiZXR0ZXIgdGhhbiB0aGlzLiBUaGUgYWRqdXN0ZWQgUi1zcXVhcmVkIGF0dGVtcHRzIHRvIA0KZG8gdGhlIHNhbWUgdGhpbmcsIGJ1dCB3ZSBkb24ndCBoYXZlIGEgc2Vuc2Ugb2Ygd2hhdCBhIG1lYW5pbmdmdWwgaW5jcmVhc2UgaW4gdGhlIA0KYWRqdXN0ZWQgUi1zcXVhcmVkIGZyb20gdGhpcyBudW1iZXIgYWxvbmUsIHNvIHdlIGFyZSBsZWZ0IHRvIGV5ZWJhbGwgaXQgYW5kIHVzZSANCm91ciBqdWRnbWVudCAtLSB3aGljaCBpcyBhY2NlcHRhYmxlIGFuZCBnb29kIGVub3VnaCBpbiBtYW55IGNhc2VzLiANCg0KQm90aCBvZiB0aGVzZSBtZWFzdXJlcyBhcmUgb25seSB0ZWxsaW5nIHVzIGhvdyB3ZWxsIG91ciBtb2RlbCBmaXRzICp0aGlzKiBkYXRhLCANCnRob3VnaC4gVGhhdCBpcywgaG93IHdlbGwgZG9lcyBvdXIgbW9kZWwgZml0IHRoaXMgc3BlY2lmaWMgc2FtcGxlIG9mIGRhdGEgdGhhdCANCml0IGhhcyBiZWVuIHNob3duLiBPZnRlbiBpbiBhcHBsaWVkIGFuYWx5c2lzLCB3ZSB3YW50IHRvIGZpbmQgYSBtb2RlbCB0aGF0IA0Kd2lsbCBmaXQgdGhpcyB5ZWFyJ3MgZGF0YSwgYnV0IGFsc28gZml0IG5leHQgeWVhcidzIGRhdGEgYXMgd2VsbC4gSW4gdGhpcyBjYXNlLCANCndlIGFyZSBpbnRlcmVzdGVkIGluIHRoZSBwcmVkaWN0aXZlIHBvd2VyIG9mIG91ciBtb2RlbCBvbiBuZXcgZGF0YS4gDQoNClRoZXJlIGFyZSBtb2RlbCBjb21wYXJpc29uIG1ldGhvZHMgdGhhdCBhbGxvdyB1cyB0byBkbyBqdXN0IHRoYXQgYnkgbG9va2luZyBhdCANCmhvdyB3ZWxsIG91ciBtb2RlbCBwcmVkaWN0cyB0aGUgZGF0YS4gV2UnbGwgZm9jdXMgb24gdGhlIHJvb3QgbWVhbiBzcXVhcmVkIA0KZXJyb3IgKFJNU0UpIC0gb25lIG9mIHRoZSBtb3N0IGNvbW1vbiBvZiB0aGVzZSBtZXRyaWNzLiBUaGUgUk1TRSBpcyB0aGUgc3F1YXJlIA0Kcm9vdCBvZiB0aGUgbWVhbiBvZiB0aGUgc3F1YXJlZCBlcnJvcnMuIFNvIHRvIGNhbGN1bGF0ZSBpdCB3ZSB0YWtlIG91ciByZXNpZHVhbHMsIA0Kc3F1YXJlIHRoZW0sIGRpdmlkZSBieSB0aGUgbnVtYmVyIG9mIHJlc2lkdWFscyAodG8gZ2V0IHRoZSBtZWFuIG9mIHRoZSBzcXVhcmVzKSwgDQphbmQgdGhlbiB0YWtlIHRoZSBzcXVhcmUgcm9vdCBvZiB0aGlzIHZhbHVlLiBJdCBwcm92aWRlcyBhIGdvb2QgYmFsYW5jZSBiZXR3ZWVuIA0KYmVpbmcgc2Vuc2l0aXZlIHRvIGxhcmdlIGVycm9ycywgYW5kIHJld2FyZGluZyBtb2RlbHMgdGhhdCBmaXQgdGhlIHdob2xlIGRhdGFzZXQgDQp3ZWxsLiBCZWNhdXNlIGl0IGlzIG1lYXN1cmluZyB0aGUgZXJyb3IsIGEgbG93ZXIgbnVtYmVyIGlzIGJldHRlci4gMCBpcyB0aGUgDQp0aGVvcmV0aWNhbCBtaW5pbXVtLiANCg0KWW91IGNhbiBmaW5kIG1hbnkgDQphbHRlcm5hdGl2ZXMgbGlrZSB0aGUgTWVkaWFuIEFic29sdXRlIEVycm9yIChNQUUpLCBNZWFuIEFic29sdXRlIERldmlhdGlvbiAoTUFEKSwgDQphbmQgbWFueSBtb3JlLiBCdXQsIFJNU0UgaXMgZmFpcmx5IHN0YW5kYXJkIGFuZCBpbXBsZW1lbnRlZCBpbiBtb3N0IHNvZnR3YXJlLiANCg0KTWVhc3VyaW5nIHByZWRpY3RpdmUgcGVyZm9ybWFuY2Ugb2YgbW9kZWxzIGl0cyBvd24gZnVsbCBndWlkZSwgYnV0IGhlcmUgYXJlIHR3byANCm1haW4gcG9pbnRzOiANCg0KMS4gQWx3YXlzIG1lYXN1cmUgcGVyZm9ybWFuY2Ugb24gZGF0YSB0aGUgbW9kZWwgaGFzIG5vdCAic2VlbiIgKGUuZy4gd2FzIG5vdCANCnVzZWQgdG8gZml0IHRoZSBtb2RlbCkuIE1vZGVscyBvdmVyZml0IHRvIHRoZSBkYXRhIHRoZXkgYXJlIGdpdmVuLCBzbyB0aGV5IA0KYXBwZWFyIG1vcmUgYWNjdXJhdGUgdGhhbiB0aGV5IHdpbGwgYmUgb24gZnV0dXJlIGRhdGEuIA0KMi4gTWVhc3VyZSB0aGUgcGVyZm9ybWFuY2UgYW5kIHZhcmlhYmlsaXR5IGluIHBlcmZvcm1hbmNlIHdoZXJlIHBvc3NpYmxlLiBXZSANCndhbnQgdG8ga25vdyBub3QganVzdCBob3cgYWNjdXJhdGUgdGhlIG1vZGVsIGlzLCBidXQgaG93IHByZWNpc2VseSB3ZSBjYW4gDQptZWFzdXJlIHRoYXQgYWNjdXJhY3kuIA0KDQpUbyBhY2hpZXZlIHRoaXMgZ29hbCB3ZSB1c2UgYSB0ZWNobmlxdWUgY2FsbGVkIHJlc2FtcGxpbmcuIFRoZXJlIGFyZSBtYW55IGZvcm1zIA0Kb2YgcmVzYW1wbGluZyAtLSBib290c3RyYXBwaW5nLCBjcm9zc3ZhbGlkYXRpb24sIGxlYXZlLW9uZS1vdXQsIGFuZCBvdGhlcnMuIFlvdSANCmNhbiBmaW5kIGV4Y2VsbGVudCB0cmVhdG1lbnRzIG9mIHRoZXNlIGNvbmNlcHRzIG9ubGluZS4gRWFjaCBvZiB0aGVtIGlzIGEgbWV0aG9kIA0KdG8gZGl2aWRlIG91ciBkYXRhIHVwIGludG8gZGF0YSB0aGUgbW9kZWwgc2VlcyAodHJhaW5pbmcgZGF0YSkgYW5kIGRhdGEgd2UgdXNlIA0KdG8gbWVhc3VyZSB0aGUgcGVyZm9ybWFuY2Ugb2YgdGhlIG1vZGVsICh0ZXN0IGRhdGEpLiBXZSBmaXQgdGhlIG1vZGVsIHRvIHRoZSANCnRyYWluaW5nIGRhdGEsIGV2YWx1YXRlIGl0IG9uIHRoZSB0ZXN0IGRhdGEsIGFuZCByZXBlYXQuIFdlIHdpbGwgZGlzY3VzcyB0aGlzIA0KbW9yZSBhdCBTZXB0ZW1iZXIgV29ya3Nob3AuDQoNClRvIGdpdmUgeW91IGEgc2Vuc2Ugb2YgdGhpcyB0ZWNobmlxdWUgbm93LCBsZXQncyB3b3JrIGEgc2ltcGxlIGV4YW1wbGUgd2hlcmUgDQp3ZSBleHRyYWN0IHRoZSBSTVNFIGZvciBvbmUgb2YgdGhlIGZpZnR5IG1vZGVscyBhbmQgZXN0aW1hdGUgaXQgZnJvbSByZXNhbXBsZWQgDQphdGEuDQoNCkZvciBub3csIHdlIHdpbGwgdXNlIGEgdGVjaG5pcXVlIGNhbGxlZCAgDQprLWZvbGQgY3Jvc3MtdmFsaWRhdGlvbiBiZWNhdXNlIGl0IGlzIGVhc3kgdG8gaW1wbGVtZW50IGFuZCBhIHN0YW5kYXJkLiANClRoaXMgbWV0aG9kICJmb2xkcyIgdGhlIGRhdGEgSyAoaGVyZSBrID0gNSkgdGltZXMgdG8gY3JlYXRlIA0KSyBkaXN0aW5jdCB0ZXN0IHNldHMgKGVhY2ggY29uc2lzdGluZyBvZiAxL2t0aCBvZiB0aGUgZGF0YSkuIFRoZW4gdGhlIG1vZGVsIA0KaXMgZml0IEsgdGltZXMsIHRvIHRoZSBkYXRhIHRoYXQgaXMgbm90IGluIGVhY2gga3RoIHRlc3Qgc2V0LCBhbmQgZXZhbHVhdGVkIA0Kb24gdGhhdCB0ZXN0IHNldC4gVGhpcyBnaXZlcyB1cyBLIGVzdGltYXRlcyBvZiB0aGUgbW9kZWwgZml0IGZyb20gb3VyIG9yaWdpbmFsIA0Kc2FtcGxlLiBJbiB0aGlzIGNhc2UsIHdlIHN0YXJ0IHdpdGggMjUxIG9ic2VydmF0aW9ucyBpbiBvdXIgZGF0YXNldCwgZm9sZCBpdCANCjUgdGltZXMsIGFuZCB0aGUgcmVzdWx0IGlzIDUgdHJhaW5pbmcgc2V0cyB3aXRoIH4yMDAgb2JzZXJ2YXRpb25zIGFuZCA1IA0KY29ycmVzcG9uZGluZyB0ZXN0IHNldHMgd2l0aCB+NTAgb2JzZXJ2YXRpb25zIHRvIG1lYXN1cmUgdGhlIG1vZGVsIHBlcmZvcm1hbmNlIA0KYWdhaW5zdCAodXNpbmcgUk1TRSBpbiB0aGlzIGNhc2UpLiANCg0KYGBge3J9DQpjdl9kYXQgPC0gY3Jvc3N2X2tmb2xkKGJ5X2dyb3VwJGRhdGFbWzddXSwgayA9IDUpDQptb2RlbHMxIDwtIG1hcChjdl9kYXQkdHJhaW4sIH5sbShzczIgfiBzczEsIGRhdGEgPSAuKSkNCm1vZGVsczIgPC0gbWFwKGN2X2RhdCR0cmFpbiwgfmxtKHNzMiB+IHNzMSArIG4xLCBkYXRhID0gLikpDQptb2RlbHMzIDwtIG1hcChjdl9kYXQkdHJhaW4sIH5sbShzczIgfiBzczEgKyBJKHNzMV4yKSArIEkoc3MxXjMpICsgSShzczFeNCkgKyBuMSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhID0gLikpDQoNCnBsb3RkZiA8LSBkYXRhLmZyYW1lKG1vZGVsXzEgPSBtYXAyX2RibChtb2RlbHMxLCBjdl9kYXQkdGVzdCwgcm1zZSksIA0KICAgICAgICAgICAgICAgICAgICAgbW9kZWxfMiA9IG1hcDJfZGJsKG1vZGVsczIsIGN2X2RhdCR0ZXN0LCBybXNlKSwNCiAgICAgICAgICAgICAgICAgICAgIG1vZGVsXzMgPSBtYXAyX2RibChtb2RlbHMzLCBjdl9kYXQkdGVzdCwgcm1zZSkpDQpwbG90ZGYNCmBgYA0KDQpUaGUgZG93bnNpZGUgb2Ygay1mb2xkIGNyb3NzLXZhbGlkYXRpb24gaXMgdGhhdCBpdCB1c2VzIGEgbG90IG9mIGRhdGEgLS0gd2UgY2FuIA0Kb25seSBnZW5lcmF0ZSBhIGZldyB1bmlxdWUgdHJhaW5pbmcvdGVzdCBzZXRzIChpbiB0aGlzIGNhc2UgNSkgd2hlbiB3ZSBvbmx5IGhhdmUgDQoyNTAgb2JzZXJ2YXRpb25zLiBUaGlzIG1ha2VzIGl0IGhhcmQgdG8gZHJhdyBjb25jbHVzaW9ucyBhYm91dCB0aGUgZGlmZmVyZW5jZSANCmJldHdlZW4gb3VyIG1vZGVscyAtIHdlIGRvbid0IGhhdmUgZW5vdWdoIGRhdGEuIExvb2sgYXQgdGhlIHJlc3VsdHMsIGl0IGxvb2tzIA0KbGlrZSBtb2RlbCAzIGdlbmVyYWxseSBoYXMgYSBsb3dlciAoYmV0dGVyKSBSTVNFLg0KDQpBcyBhbiBhbHRlcm5hdGl2ZSwgZm9yIGNhc2VzIHdoZXJlIHdlIGRvIG5vdCBoYXZlIGEgbG90IG9mIGRhdGEgdG8gZ2VuZXJhdGUgDQplbm91Z2ggaG9sZG91dCBzZXRzLCB3ZSBjYW4gdXNlIGEgbW9yZSBnZW5lcmFsIHRlY2huaXF1ZSBjYWxsZWQgYm9vdHN0cmFwcGluZy4gDQpJbiB0aGlzIGNhc2UsIHdlIGZpdCBvdXIgbW9kZWwgdG8gdGhlIGZ1bGwgc2FtcGxlIC0gc28gd2UgZ2V0IHRoZSBiZW5lZml0IG9mIGFsbCANCnRoZSBvYnNlcnZhdGlvbnMuIFRoZW4sIHdlIGV2YWx1YXRlIGl0cyBwZXJmb3JtYW5jZSBhZ2FpbnN0IE4gcmVzYW1wbGVzIGNyZWF0ZWQgDQpieSBzYW1wbGluZyB0aGUgZGF0YSB3aXRoIHJlcGxhY2VtZW50LiBTbywgaW4gdGhpcyBjYXNlLCB3ZSB3aWxsIGNyZWF0ZSAxMDAgDQpyZXNhbXBsZXMgb2YgdGhlIHNhbWUgc2l6ZSBhcyB0aGUgZnVsbCBzYW1wbGUsIGRyYXduIGZyb20gdGhlIGZ1bGwgc2FtcGxlIGJ1dCANCmRyYXduIHdpdGggcmVwbGFjZW1lbnQuIFRoaXMgd2lsbCBnaXZlIHVzIDEwMCBlc3RpbWF0ZXMgb2YgdGhlIFJNU0UgaW5zdGVhZCANCm9mIDUuIA0KDQoNCmBgYHtyfQ0KYm9vdF9kYXQgPC0gYm9vdHN0cmFwKGJ5X2dyb3VwJGRhdGFbWzddXSwgbiA9IDEwMCkgIyBib290c3RyYXAgc2FtcGxlcw0KbW9kZWxzMSA8LSBsbShzczIgfiBzczEsIGRhdGEgPSBieV9ncm91cCRkYXRhW1s3XV0pICMgZml0IG1vZGVsIDEgdG8gb3JpZ2luYWwgc2FtcGxlDQptb2RlbHMyIDwtIGxtKHNzMiB+IHNzMSArIG4xLCBkYXRhID0gYnlfZ3JvdXAkZGF0YVtbN11dKQ0KbW9kZWxzMyA8LSBsbShzczIgfiBzczEgKyBJKHNzMV4yKSArIEkoc3MxXjMpICsgSShzczFeNCkgKyBuMSwgDQogICAgICAgICAgICAgIGRhdGEgPSBieV9ncm91cCRkYXRhW1s3XV0pDQoNCg0KcGxvdGRmIDwtIGRhdGEuZnJhbWUobW9kZWxfMSA9IG1hcF9kYmwoYm9vdF9kYXQkc3RyYXAsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfnJtc2UobW9kZWxzMSwgZGF0YSA9IC54KSksICMgY2FsY3VsYXRlIFJNU0UNCiAgICAgICAgICAgICAgICAgICAgICMgb24gZWFjaCBzdWJzYW1wbGUNCiAgICAgICAgICAgICAgICAgICAgIG1vZGVsXzIgPSBtYXBfZGJsKGJvb3RfZGF0JHN0cmFwLCB+cm1zZShtb2RlbHMyLCBkYXRhID0gLngpKSwNCiAgICAgICAgICAgICAgICAgICAgIG1vZGVsXzMgPSBtYXBfZGJsKGJvb3RfZGF0JHN0cmFwLCB+cm1zZShtb2RlbHMzLCBkYXRhID0gLngpKSkNCg0Kc3VtbWFyeShwbG90ZGYpDQpgYGANCg0KV2l0aCAxMDAgbWVhc3VyZXMgb2YgUk1TRSBpdCBsb29rcyBjbGVhciB0aGF0IG91ciB0aGlyZCBtb2RlbCBpcyBzdXBlcmlvciAtIGl0IA0KaGFzIGEgbG93ZXIgYXZlcmFnZSBSTVNFIGFuZCBpdCBoYXMgYSBsb3dlciBtYXhpbXVtIGFuZCBtaW5pbXVtIHZhbHVlcyBvZiBSTVNFIA0KYXMgd2VsbC4gWW91IGNhbiB1c2UgRURBIHRlY2huaXF1ZXMgdG8gcGxvdCB0aGVzZSBkaWZmZXJlbmNlcyBhcyB3ZWxsIGlmIHlvdSANCmxpa2UuIA0KDQpUaGlzIGJyaWVmIGludHJvZHVjdGlvbiBoYXMgZGVtb25zdHJhdGVkIHRocmVlIHBvc3NpYmxlIHdheXMgeW91IGNhbiBjb25kdWN0IA0KbW9kZWwgY29tcGFyaXNvbjoNCg0KMS4gSW5mb3JtYWxseSwgaG93IHdlbGwgZG9lcyBlYWNoIG1vZGVsIGV4cGxhaW4gdGhlIHZhcmlhbmNlIGluIHRoZSBkYXRhIGl0IGlzIA0KZml0IHRvIChSLXNxdWFyZWQpDQoyLiBEaXJlY3RseSBhbmQgZm9ybWFsbHksIGRvZXMgdGhlIGRhdGEgc3VwcG9ydCB0aGUgZXZpZGVuY2UgdGhhdCBvbmUgbW9kZWwgZml0cyANCnRoZSBkYXRhIGJldHRlciB0aGFuIGFub3RoZXIgbW9kZWwgKEYtdGVzdHMpDQozLiBEaXJlY3RseSBhbmQgaW5mb3JtYWxseSwgZG9lcyBvbmUgbW9kZWwgaGF2ZSBhIGxvd2VyIHByZWRpY3RpdmUgZXJyb3Igb24gDQpkYXRhIGl0IGhhcyBub3Qgc2VlbiBiZWZvcmUgdGhhbiB0aGUgb3RoZXIgbW9kZWwgKHJlc2FtcGxpbmcgKyBSTVNFKQ0KDQpXZSB3aWxsIGRldmVsb3AgYW5kIGV4dGVuZCB0aGVzZSBpZGVhcyBvZiBwcmVkaWN0aXZlIHBlcmZvcm1hbmNlIG9mIG91ciBtb2RlbHMgaW4gDQpmdXR1cmUgd29ya3Nob3BzIGFuZCB0dXRvcmlhbHMuIA0KDQojIyBDb21tdW5pY2F0aW5nIFJlZ3Jlc3Npb24gTW9kZWxzIHdpdGggR3JhcGhpY3MNCg0KQXMgZGlzY3Vzc2VkIGluIHRoZSBsZWN0dXJlLCBpbnRlcnByZXRhdGlvbiBvZiByZWdyZXNzaW9uIG1vZGVscyBjYW4gYmUgdmVyeQ0KZGlmZmljdWx0IGFzIG1vZGVscyBtb3ZlIGJleW9uZCBzaW1wbGUgYmktdmFyaWF0ZSByZWxhdGlvbnNoaXBzLiBXZSBjYW4gbG9vayBhdA0KY29lZmZpY2llbnRzIGFuZCBtb2RlbCBmaXQgc3RhdGlzdGljcywgYnV0IHRoaXMgaXMgbGlrZSBsb29raW5nIGF0IHRoZSBnZWFycyBvZg0KYSBtYWNoaW5lIGFuZCB0cnlpbmcgdG8gcGljdHVyZSBob3cgaXQgd29ya3MuIEl0J3MgZWFzaWVyIGlmIHdlIGp1c3QgdHVybiBpdCBvbiENCkFuZCByZWdyZXNzaW9uIG1vZGVscyBhcmUgY29tcHV0YXRpb25hbCBtYWNoaW5lcyBkZXNpZ25lZCB0byBtYWtlIHByZWRpY3Rpb25zLA0Kc28gd2UgY2FuIHVzZSB0aGF0IGZlYXR1cmUgdG8gc2ltdWxhdGUgdmFsdWVzIGZvciB0aGVtIGFuZCB3YXRjaCBob3cgdGhleQ0KYmVoYXZlLg0KDQpGb3J0dW5hdGVseSwgdGhpcyBwb3dlcmZ1bCB0ZWNobmlxdWUgYnVpbGRzIG9uIHNraWxscyB3ZSBoYXZlIGFscmVhZHkgZGV2ZWxvcGVkDQpmb3IgZXhwbG9yYXRvcnkgZGF0YSBhbmFseXNpcy4gVGhlIG9ubHkgZGlmZmVyZW5jZSBpcyB0aGF0IGhlcmUgdGhlIGRhdGEgd2UgYXJlDQpleHBsb3JpbmcgYXJlIGhvdyBvdXIgbW9kZWwgcHJlZGljdHMgb3V0Y29tZXMgaW4gcmVzcG9uc2UgdG8gZGlmZmVyZW50IGlucHV0IHdlDQpnaXZlIGl0Lg0KDQpMZXQncyBsb29rIGF0IHNvbWUgZXhhbXBsZXMuIA0KDQpGaXJzdCwgbGV0J3MgY3JlYXRlIGEgY29lZmZpY2llbnQgcGxvdCBhbmQgZ2F6ZSB1cG9uIHRoZSBnZWFycyBvZiBvdXIgbW9kZWw6IA0KDQpgYGB7cn0NCmxpYnJhcnkoY29lZnBsb3QpICMgZWFzeSBwYWNrYWdlIGZvciB2aXN1YWxpemluZyBsbSBjb2VmZmljaWVudHMNCmNvZWZwbG90KG0zLCBpbnRlcmNlcHQgPSBGQUxTRSkgKyAgIyBpbnRlcmNlcHQgPSBGQUxTRSBhdm9pZHMgc2tld2luZyB0aGUgcGxvdA0KICAjIGJlY2F1c2Ugb2YgdGhlIGxhcmdlIG1hZ25pdHVkZSBvZiB0aGUgaW50ZXJjZXB0DQogIHRoZW1lX2J3KCkNCmBgYA0KDQpUaGlzIGdyYXBoaWMgaXMgbm90IHZlcnkgZW5saWdodGVuaW5nLiBEbyB5b3Uga25vdyB3aHk/IElmIHdlIHdyb3RlIG91dCB0aGUgDQplcXVhdGlvbiBmb3Igb3VyIHJlZ3Jlc3Npb24gbW9kZWwsIHdlIHdvdWxkIHNlZSB0aGF0IGJlY2F1c2Ugb2YgdGhlIHNjYWxlIG9mIG91ciANCnByZWRpY3RvciB2YXJpYWJsZXMgKGFuZCB0aGUgYHNzMWAgcHJlZGljdG9yIHNwZWNpZmljYWxseSkgYW5kIG91ciBvdXRjb21lLCANCnRoZSBjb2VmZmljaWVudHMgaGF2ZSB0byBoYXZlIHZlcnkgbGFyZ2UgYW5kIHZlcnkgc21hbGwgdmFsdWVzIHRvIGZpdCB0aGUgZGF0YSANCndlbGwgd2hlbiBpbmNvcnBvcmF0aW5nIHBvbHlub21pYWwgdGVybXMuIFVzaW5nIHBvbHlub21pYWwgdGVybXMgcmVzdWx0cyBpbiANCm1ha2luZyBvdXIgY29lZmZpY2llbnRzIG11Y2ggaGFyZGVyIHRvIGludGVycHJldCBiZWNhdXNlIG9mIHRoZSBtYWduaXR1ZGVzIG9mIA0KdmFyaWFibGVzIGludm9sdmVkLiANCg0KYGBge3J9DQpjb2VmcGxvdChtMiwgaW50ZXJjZXB0ID0gRkFMU0UpICsgdGhlbWVfYncoKQ0KYGBgDQoNCkhlcmUncyBhIHNpbXBsZXIgcGxvdCBjb3JyZXNwb25kaW5nIHRvIGEgc2ltcGxlciBtb2RlbC4gDQoNCkJ1dCB3ZSBkb24ndCB3YW50IHRvIGJlIGxpbWl0ZWQgaW4gb3VyIGludGVycHJldGF0aW9uIHRvIHNpbXBsZSBtb2RlbHMsIHNvIGxldCdzIA0KbG9vayBhdCB0ZWNobmlxdWVzIHRoYXQgYWxsb3cgdXMgdG8gaW50ZXJwcmV0IGFuZCB1bmRlcnN0YW5kIG1vcmUgY29tcGxleCBtb2RlbHMgDQphbmQgZ28gYmV5b25kIHRoZSB0cmFkaXRpb25hbCByZXBvcnRpbmcgb2YgY29lZmZpY2llbnRzLiANCg0KIyMjIE1hcmdpbiBQbG90cw0KDQpNYXJnaW4gcGxvdHMgYXJlIGdyYXBoaWNzIHRoYXQgbG9vayBhdCBob3cgdGhlIHByZWRpY3RvciBvciB0aGUgY29lZmZpY2llbnQgb2YgDQppbnRlcmVzdCBiZWhhdmVzIGZvciBjZXJ0YWluIHN1YnNldHMgb2YgdGhlIGRhdGEuIFRoaXMgaGVscHMgdXMgbG9vayBhdCBob3cgdGhlIA0KbW9kZWwgYmVoYXZpb3IgY2hhbmdlcyBhcyB2YWx1ZXMgb2Yga2V5IHZhcmlhYmxlcyBpbiBvdXIgZGF0YSBjaGFuZ2UuIFRoaXMgaXMgYSANCnBvd2VyZnVsIHRlY2huaXF1ZSB1c2VmdWwgZm9yIGV4cGxvcmluZyBjb21wbGV4IG1vZGVsIGJlaGF2aW9ycy4gTGV0J3MgcmV0dXJuIA0KdG8gb3VyIHBvbHlub21pYWwgbW9kZWw6DQoNCmBgYHtyfQ0KbGlicmFyeShtYXJnaW5zKSAjIHBhY2thZ2UgdG8gY29tcHV0ZSBtYXJnaW5hbCBwbG90cw0KbWFyZ2lucyhtMykNCm1hcmdpbmFsX2VmZmVjdHMobTMpDQoNCiMgU3B1cmlvdXMNCm1hcmdpbnM6OmNwbG90KG0zLCAic3MxIikNCg0KYGBgDQoNClRoaXMgY2xlYXJseSBzaG93cyB1cyB0aGUgcG9seW5vbWlhbCBzaGFwZSB0aGF0IHRoZSBhZGRpdGlvbmFsIHRlcm1zIGFyZSANCmxldHRpbmcgdXMgZXhwbG9yZS4gTGV0J3MgbG9vayBhdCBhbm90aGVyIGV4YW1wbGUgLSB0aGlzIHRpbWUgbGV0J3MgbG9vayBhdCBob3cgDQp3ZSBjYW4gdXNlIG1hcmdpbmFsIHBsb3RzIHRvIGV4cGxvcmUgYSBtb2RlbCB0aGF0IGluY2x1ZGVzIGNhdGVnb3JpY2FsIHZhcmlhYmxlcyANCmFuZCBhbiBpbnRlcmFjdGlvbiBiZXR3ZWVuIHRob3NlIHZhcmlhYmxlcyBhbmQgdGhlIG91dGNvbWUuIA0KDQpgYGB7cn0NCm00IDwtIGxtKHNzMiB+IHNzMSArIHNzMSAqIGxvY2FsZV9kZXNjLCBkYXRhID0gZnVsbF9kYXRhKQ0KY29lZnBsb3QobTQsIGludGVyY2VwdCA9IEZBTFNFKSArIHRoZW1lX2J3KCkNCm1hcmdpbnM6OmNwbG90KG00LCAic3MxIikgIyBhdmVyYWdlIGVmZmVjdCBvZiBzczENCmBgYA0KDQpgYGB7cn0NCm1hcmdpbnM6OmNwbG90KG00LCAibG9jYWxlX2Rlc2MiKSAjIGF2ZXJhZ2UgZWZmZWN0IG9mIGxvY2FsZQ0KYGBgDQoNCg0KYGBge3J9DQptYXJnaW5zOjpjcGxvdChtNCwgImxvY2FsZV9kZXNjIiwgZHggPSAic3MxIiwgd2hhdCA9ICJlZmZlY3QiKSAjIGtlZXANCmBgYA0KVGhlIGFib3ZlIGZpZ3VyZSBzaG93cyB0aGUgcHJlY2lzaW9uIGFuZCB2YXJpYWJpbGl0eSBpbiBlc3RpbWF0ZXMgb2YgYHNzMWAgZm9yIA0KZWFjaCBsb2NhbGUsIGJhc2VkIG9uIHRoZSBpbnRlcmFjdGlvbiB0ZXJtLiBUaGVzZSBlZmZlY3RzIGFyZSBwbG90dGVkIG9uIHRoZSANCnNjYWxlIG9mIHRoZSBgc3MxYCBjb2VmZmljaWVudCBhbmQgcmVwcmVzZW50IHRoZSBzdW0gb2YgdGhlIGBzczFgIGNvZWZmaWNpZW50IA0KYW5kIHRoZSBpbnRlcmFjdGlvbiBlZmZlY3QgLSB0aGlzIGlzIGEgbW9yZSBkaXJlY3Qgd2F5IHRvIGludGVycHJldCBpbnRlcmFjdGlvbiANCmVmZmVjdHMgdGhhbiB0aGUgYGNvZWZwbG90YCBvciB0YWJsZSBvZiBjb2VmZmljaWVudHMuIA0KDQpXZSBjYW4gYWxzbyBwbG90IG11bHRpcGxlIGxpbmVzLCBvbmUgZm9yIGVhY2ggbG9jYWxlLCB0byBsb29rIGF0IGhvdyB0aGUgbGluZSANCmlzIGNoYW5nZWQgYnkgZWFjaCBsb2NhbGUuIFRoaXMgaXNuJ3QgdGhlIG9ubHkgd2F5IHRvIHBsb3QgbXVsdGlwbGUgbGluZXMgZnJvbSANCnRoaXMgbW9kZWwsIGJ1dCB5b3UgY2FuIGRvIHRoZSBjb21wdXRhdGlvbiB3aXRoIHRoZSBtYXJnaW4gcGFja2FnZSBhbmQgYWRkIHRoZSANCmxpbmVzIHRvZ2V0aGVyIG9uIHRoZSBzYW1lIHBsb3Qgd2l0aCB0aGlzIGNvZGU6DQoNCmBgYHtyfQ0KbG9jYWwoew0KICBjcGxvdChtNCwgInNzMSIsIGRhdGEgPSBmdWxsX2RhdGFbZnVsbF9kYXRhJGxvY2FsZV9kZXNjID09ICJSdXJhbCwgb3V0c2lkZSBNU0EiLF0pDQogIGNwbG90KG00LCAic3MxIiwgZGF0YSA9IGZ1bGxfZGF0YVtmdWxsX2RhdGEkbG9jYWxlX2Rlc2MgPT0gIk1pZC1TaXplIENpdHkiLF0sIA0KICAgICAgICBkcmF3ID0gImFkZCIsIGNvbCA9ICJibHVlIikNCiAgY3Bsb3QobTQsICJzczEiLCBkYXRhID0gZnVsbF9kYXRhW2Z1bGxfZGF0YSRsb2NhbGVfZGVzYyA9PSAiU21hbGwgVG93biIsXSwgDQogICAgICAgIGRyYXcgPSAiYWRkIiwgY29sID0gInJlZCIpDQogIGNwbG90KG00LCAic3MxIiwgZGF0YSA9IGZ1bGxfZGF0YVtmdWxsX2RhdGEkbG9jYWxlX2Rlc2MgPT0gIlVyYmFuIEZyaW5nZSBvZiBhIExhcmdlIENpdHkiLF0sIA0KICAgICAgICBkcmF3ID0gImFkZCIsIGNvbCA9ICJwdXJwbGUiKQ0KICBjcGxvdChtNCwgInNzMSIsIGRhdGEgPSBmdWxsX2RhdGFbZnVsbF9kYXRhJGxvY2FsZV9kZXNjID09ICJNaWQtU2l6ZSBDaXR5IixdLCANCiAgICAgICAgZHJhdyA9ICJhZGQiLCBjb2wgPSAiZ29sZCIpDQogIGNwbG90KG00LCAic3MxIiwgZGF0YSA9IGZ1bGxfZGF0YVtmdWxsX2RhdGEkbG9jYWxlX2Rlc2MgPT0gIkxhcmdlIENpdHkiLF0sIA0KICAgICAgICBkcmF3ID0gImFkZCIsIGNvbCA9ICJvcmFuZ2UiKQ0KfSkNCg0KYGBgDQoNCg0KVGhlIGFkdmFudGFnZSBvZiBtYXJnaW5hbCBwbG90cyBpcyB0aGV5IGFsbG93IHlvdSB0byBpbnRlcnByZXQgdGhlIG1vZGVsIG9uIHRoZSANCnNjYWxlIG9mIHRoZSBvdXRjb21lLCBub3Qgb24gdGhlIHNjYWxlIG9mIHRoZSBjb2VmZmljaWVudHMuIFRoaXMgY2FuIGJlIG11Y2ggDQptb3JlIGludHVpdGl2ZSB0byB1bmRlcnN0YW5kIGFuZCBhbGxvdyB5b3UgdG8gcHJlc2VudCB0byBhdWRpZW5jZXMgdGhhdCBhcmUgDQpmYW1pbGlhciB3aXRoIHRoZSBvdXRjb21lIHZhcmlhYmxlLCBidXQgbm90IG5lY2Vzc2FyaWx5IHRoZSBpbm5lci13b3JraW5ncyBvZiANCnJlZ3Jlc3Npb24gbW9kZWxzLiANCg0KIyMjIENvdW50ZXJmYWN0dWFsIFBsb3RzDQoNCkEgZmluYWwgdmlzdWFsIHdheSB0byBleHBsb3JlIG91ciBkYXRhIGlzIHRvIGNyZWF0ZSBjb3VudGVyZmFjdHVhbHMgYW5kIHNlZSBob3cNCnJlZmVyZW5jZSBvYnNlcnZhdGlvbnMgb3IgcmFuZ2VzIG9mIHByZWRpY3RvcnMgY2hhbmdlIGFjcm9zcyBvdGhlciB2YWx1ZXMuIExldCdzDQpjb250aW51ZSB3aXRoIGxvb2tpbmcgYXQgbG9jYWxlIGJhc2VkIG9uIG91ciBiZWxpZWYgdGhhdCBkaWZmZXJlbnQgZ2VvZ3JhcGhpYw0KbG9jYXRpb25zIGhhdmUgZGlmZmVyZW50IGdyb3d0aCBmdW5jdGlvbnMgb24gdGVzdCBzY29yZXMgLS0gcGVyaGFwcyBkdWUgdG8gdGhlDQpsb2NhbCB0ZWFjaGluZyBsYWJvciBtYXJrZXQgb3Igc29tZSBvdGhlciBjb21tdW5pdHkgZmFjdG9ycy4NCg0KTHVja2lseSwgY291bnRlcmZhY3R1YWwgcGxvdHMgYXJlIGVhc3kgdG8gbWFrZS4gV2UganVzdCBoYXZlIHRvIGdldCBjb21mb3J0YWJsZQ0KbWFraW5nIGZha2UgZGF0YSENCg0KU28gbGV0J3MgbG9vayBhdCBob3cgdGhlIHByZWRpY3RlZCBzY29yZSBvZiBhbiBvYnNlcnZhdGlvbiB3b3VsZCBjaGFuZ2UgYmFzZWQgb24NCml0IGJlaW5nIGFzc2lnbmVkIHRvIGEgZGlmZmVyZW50IGxvY2FsZS4gVGhpcyBpcyBhIHZlcnkgaHlwb3RoZXRpY2FsDQpjb3VudGVyZmFjdHVhbCwgc2luY2Ugd2UgY2Fubm90IG1vdmUgc2Nob29scyBmcm9tIG9uZSBnZW9ncmFwaGljIHJlZ2lvbiB0bw0KYW5vdGhlciwgYnV0IGl0IGlzIGFuIGltcG9ydGFudCBvbmUgZm9yIGhpZ2hsaWdodGluZyBob3cgb3VyIG1vZGVsIHZpZXdzIHRoZQ0KZGF0YS4NCg0KYGBge3J9DQptNSA8LSBsbSh1cGRhdGUoZm9ybXVsYShtMyksIC4gfiAuICsgc3MxICogbG9jYWxlX2Rlc2MpLCBkYXRhID0gZnVsbF9kYXRhKQ0KIyBQaWNrIGEgcm93LCBoZXJlIHdlIHNhbXBsZSBhdCByYW5kb20NCmV4YW1wbGVfc2Nob29sIDwtIGZ1bGxfZGF0YVtzYW1wbGUocm93Lm5hbWVzKGZ1bGxfZGF0YSksIDEpLCBdDQpleGFtcGxlX3NjaG9vbCA8LSBleGFtcGxlX3NjaG9vbFtyZXAoc2VxX2xlbihucm93KGV4YW1wbGVfc2Nob29sKSksDQogICAgICAgICAgICAgICAgICAgZWFjaD1sZW5ndGgodW5pcXVlKGZ1bGxfZGF0YSRsb2NhbGVfZGVzYykpKSxdDQpleGFtcGxlX3NjaG9vbCRsb2NhbGVfZGVzYyA8LSB1bmlxdWUoZnVsbF9kYXRhJGxvY2FsZV9kZXNjKQ0KZXhhbXBsZV9zY2hvb2wkeWhhdCA8LSBwcmVkaWN0KG01LCBuZXdkYXRhID0gZXhhbXBsZV9zY2hvb2wpDQoNCmdncGxvdChleGFtcGxlX3NjaG9vbCwgYWVzKHggPSBsb2NhbGVfZGVzYywgeSA9IHloYXQpKSArIA0KICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSBleGFtcGxlX3NjaG9vbCRzczJbMV0sIGNvbG9yID0gSSgicmVkIikpICsNCiAgZ2VvbV9wb2ludCgpICsgdGhlbWVfYncoKSArIA0KICB0aGVtZShheGlzLnRleHQgPSBlbGVtZW50X3RleHQoYW5nbGUgPSAyNSkpDQogIA0KYGBgDQoNCkZyb20gdGhpcyBwbG90IHdlIGNhbiBzZWUgdGhhdCB0aGUgb2JzZXJ2ZWQgdmFsdWUgKHJlZCBob3Jpem9udGFsIGxpbmUpIGRldmlhdGVzIA0KdGhhbiBhbnkgcHJlZGljdGlvbiBubyBtYXR0ZXIgdGhlIGxvY2FsZSB3ZSBhcmUgaW4uIFdlIGNhbiBhbHNvIHNlZSB0aGF0IHRoZSANCm1vZGVsIGJlbGlldmVzIG91ciBzY2hvb2wgd291bGQgZmFyZSBwb29ybHkgaW4gYSBMYXJnZSBDaXR5LCBidXQgZG8gYmVzdCBpZiANCml0IHdhcyBydXJhbCwgYnV0IGluc2lkZSBhbiBNU0EuIA0KDQojIyBSb2J1c3QgYW5kIFdlaWdodGVkIFJlZ3Jlc3Npb24NCg0KT25lIHByb2JsZW0gd2UgaGF2ZSBub3RlZCB3aXRoIG91ciBkYXRhIHByZXZpb3VzbHkgaXMgdGhhdCBpdCB0cmVhdHMgYmlnIGFuZCANCnNtYWxsIHNjaG9vbHMgYWxpa2UuIEZvciBzb21lIGFuYWx5c2VzIHRoaXMgaXMgZmluZSwgYnV0IGZvciBvdGhlcnMsIGl0IG1heSANCmNhdXNlIG91ciBhbmFseXNpcyBwcm9ibGVtcy4gSW4gdGhlc2UgY2FzZXMsIHdlIGNhbiB3ZWlnaHQgb3VyIG1vZGVsIGJ5IHRoZSANCm51bWJlciBvZiBzdHVkZW50cyBpbiBhIHNjaG9vbCBzbyB0aGF0IG9ic2VydmF0aW9ucyB3aXRoIGxhcmdlIG51bWJlcnMgb2Ygc2Nob29scyANCndpbGwgYmUgImNvdW50ZWQgbW9yZSIgYnkgdGhlIHJlZ3Jlc3Npb24gbW9kZWxzIHRoYW4gb2JzZXJ2YXRpb25zIHdpdGggZmV3ZXIgDQpzY2hvb2xzLiANCg0KQ29tcGFyaW5nIHdlaWdodGVkIGFuZCB1bndlaWdodGVkIG1vZGVscyBpcyBub3Qgc29tZXRoaW5nIHRoYXQgY2FuIGJlIGRvbmUgd2l0aCANCmFuIEYtdGVzdC4gSG93ZXZlciwgYnkgdXNpbmcgcHJlZGljdGl2ZSBtb2RlbCBmaXQgbWV0cmljcyBsaWtlIHRoZSBSTVNFIHRoaXMgaXMgDQpwb3NzaWJsZS4gTGV0J3Mgc2VlIGhvdyBvdXIgcG9seW5vbWlhbCBtb2RlbCBjaGFuZ2VzIHdoZW4gd2Ugd2VpZ2h0IGJ5IHRoZSANCm51bWJlciBvZiBzdHVkZW50cyBpbiBhIGNvaG9ydC4gDQoNCg0KYGBge3J9DQp1bndlaWdodGVkX21vZCA8LSBsbShzczIgfiBzczEgKyBJKHNzMV4yKSArIEkoc3MxXjMpICsgSShzczFeNCksIA0KICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhID0gYnlfZ3JvdXAkZGF0YVtbN11dKQ0Kd2VpZ2h0ZWRfbW9kIDwtIGxtKHNzMiB+IHNzMSArIEkoc3MxXjIpICsgSShzczFeMykgKyBJKHNzMV40KSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBieV9ncm91cCRkYXRhW1s3XV0sIHdlaWdodHMgPSBuMSkNCg0KDQpwbG90ZGYgPC0gZGF0YS5mcmFtZSh1bndlaWdodF9ybXNlID0gbWFwX2RibChib290X2RhdCRzdHJhcCwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB+cm1zZSh1bndlaWdodGVkX21vZCwgZGF0YSA9IC54KSksIA0KICAgICAgICAgICAgICAgICAgICAgd2VpZ2h0X3Jtc2UgPSBtYXBfZGJsKGJvb3RfZGF0JHN0cmFwLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH5ybXNlKHdlaWdodGVkX21vZCwgZGF0YSA9IC54KSkpDQogICAgICAgICAgICAgICAgDQpnZ3Bsb3QocGxvdGRmLCBhZXMoeCA9IHVud2VpZ2h0X3Jtc2UpKSArIA0KICBnZW9tX2RlbnNpdHkoY29sb3IgPSBJKCJwdXJwbGUiKSkgKyANCiAgZ2VvbV9kZW5zaXR5KGFlcyh4ID0gd2VpZ2h0X3Jtc2UpLCBjb2xvciA9IEkoImRhcmtncmVlbiIpKSArIA0KICB0aGVtZV9idygpICsgbGFicyh4ID0gIlJNU0UiKQ0KYGBgDQoNClRoZSB3ZWlnaHRlZCBSTVNFIChkYXJrIGdyZWVuKSBoYXMgYSBoaWdoZXIgZGlzdHJpYnV0aW9uIG92ZXIgb3VyIGJvb3RzdHJhcA0KcmVzYW1wbGVzIHRoYW4gdGhlIHVud2VpZ2h0ZWQgUk1TRSAocHVycGxlKS4gUmVtZW1iZXIgdGhhdCB3ZSBjYW4gaW50ZXJwcmV0IHRoZQ0KUk1TRSBhcyBiZWluZyBvbiB0aGUgc2NhbGUgb2YgdGhlIHNxdWFyZSByb290IG9mIHRoZSBvdXRjb21lLiBPdXIgb3V0Y29tZSBoYXMgYQ0KbWVhbiBvZiB+MTEwMCAoc3F1YXJlIHJvb3QgfjMzKSAuIE91ciBhdmVyYWdlIHByZWRpY3RlZCBvYnNlcnZhdGlvbiBlcnJvciB0aGVuDQpyZXByZXNlbnRzIGEgc3Vic3RhbnRpYWwgZnJhY3Rpb24gb2YgdGhlIG1lYW4gb2Ygb3VyIG91dGNvbWUuDQoNCkNvbXBhcmUgdHdvIGV4YW1wbGUgbW9kZWxzIHRvIGdldCBhIHNlbnNlIG9mIGhvdyBtdWNoIHRoZSB3ZWlnaHRzIGNoYW5nZSB0aGluZ3MuDQoNCmBgYHtyfQ0KbWFyZ2luczo6Y3Bsb3Qod2VpZ2h0ZWRfbW9kLCAic3MxIiwgY29sID0gImRhcmtncmVlbiIpDQptYXJnaW5zOjpjcGxvdCh1bndlaWdodGVkX21vZCwgInNzMSIsIGRyYXcgPSAiYWRkIiwgY29sID0gInB1cnBsZSIpDQpgYGANCg0KVGhlIGNvZWZmaWNpZW50cyBjaGFuZ2UgKmEgbG90KiBiZXR3ZWVuIHRoZXNlIHR3byBtb2RlbHMsIGp1c3QgYnkgbW9kaWZ5aW5nIHRoZSANCndlaWdodHMuIEJ1dCwgd2hlbiB3ZSBwbG90IHRoZXNlIG1vZGVscyB3ZSBjYW4gc2VlIHRoYXQgdGhleSBhcmUgc3RpbGwgbGFyZ2VseSANCnRoZSBzYW1lIG9uIHRoZSBzY2FsZSBvZiB0aGUgb3V0Y29tZS4gDQoNCiMjIyBSb2J1c3QgUmVncmVzc2lvbg0KDQpUaG9zZSBvZiB5b3Ugd2hvIGhhdmUgdXNlZCBTdGF0YSBtYXkgYmUgd29uZGVyaW5nIHdoZXJlIGByb2J1c3RgIHJlZ3Jlc3Npb24NCmVzdGltYXRlcyBjb21lIGludG8gcGxheS4gUm9idXN0IHJlZ3Jlc3Npb24gY29tZXMgaW4gdHdvIGZvcm1zOiB0aGUgZmlyc3QgaXMNCnJvYnVzdCByZWdyZXNzaW9uIC0gYSByZWdyZXNzaW9uIHRlY2huaXF1ZSB0aGF0IHJlc3VsdHMgaW4gZGlmZmVyZW50DQoqY29lZmZpY2llbnRzKiBhbmQgZGlmZmVyZW50ICpzdGFuZGFyZCBlcnJvcnMqLiBSb2J1c3QgcmVncmVzc2lvbiBpcyBkZXNpZ25lZCB0bw0KYmUgbW9yZSByZXNpc3RhbnQgdG8gdGhlIGluZmx1ZW5jZSBvZiBleHRyZW1lIG91dGxpZXIgb2JzZXJ2YXRpb25zLCBzbyBpdCBjYW4gYmUNCmEgdXNlZnVsIHRlY2huaXF1ZSBmb3IgYW5hbHl6aW5nIGRhdGEgd2l0aCBvdXRsaWVycy4gSXQgYWNoaWV2ZXMgdGhpcyBieSB1c2luZyBhDQpkaWZmZXJlbnQgbWV0aG9kIGZvciBjYWxjdWxhdGluZyB0aGUgbGVhc3Qtc3F1YXJlcyBlcXVhdGlvbi4gVGhpcyBtYWtlcyBpdA0KZGlmZmVyZW50IGZyb20gdGhlICoqc2Vjb25kKiogZm9ybSBvZiByb2J1c3RuZXNzLCBhIGNvcnJlY3Rpb24gb2YgdGhlIHN0YW5kYXJkDQplcnJvcnMuDQoNCkluIHByYWN0aWNlLCB0aGUgY2hhbmdlIGluIGNvZWZmaWNpZW50cyBmcm9tIHJvYnVzdCByZWdyZXNzaW9uIHdpbGwgb2Z0ZW4gYmUgDQpzbWFsbC4gRm9yIG91ciBleGFtcGxlIGhlcmUsIHdlIHdpbGwgZml0IG91ciBtb2RlbCB0byB0aGUgZnVsbCBkYXRhIHNldCwgbm90IA0Kb25lIG9mIG91ciBmaWZ0eSBzdWJzYW1wbGVzLiANCg0KYGBge3J9DQpzb3VyY2UoIlIvcm9idXN0X2Z1bmN0aW9ucy5SIikNCmxpYnJhcnkoTUFTUykNCmxpYnJhcnkoc2FuZHdpY2gpDQpsaWJyYXJ5KGxtdGVzdCkNCg0KZnVsbF9kYXRhJGZybF9wZXJbaXMubmEoZnVsbF9kYXRhJGZybF9wZXIpXSA8LSAwDQptX25vdHJvYnVzdCA8LSBsbShzczIgfiBzczEgKyBuMSArIHRvdGFsX2Vucm9sbG1lbnQgKyBmcmxfcGVyICsgYXR0X3JhdGUgKyANCiAgICAgICAgICAgICAgICAgICAgZmFjdG9yKGdyYWRlKSArIHN1YmplY3QgKyBmYWN0b3IodGVzdF95ZWFyKSwgDQogICAgICAgICAgICAgICAgICBkYXRhID0gZnVsbF9kYXRhKQ0KbV9yb2J1c3QgPC0gcmxtKHNzMiB+IHNzMSArIG4xICsgdG90YWxfZW5yb2xsbWVudCArIGZybF9wZXIgKyBhdHRfcmF0ZSArIA0KICAgICAgICAgICAgICAgICAgZmFjdG9yKGdyYWRlKSArIHN1YmplY3QgKyBmYWN0b3IodGVzdF95ZWFyKSwgDQogICAgICAgICAgICAgICAgICBkYXRhID0gZnVsbF9kYXRhKQ0KDQpjb2VmcGxvdDo6bXVsdGlwbG90KG1fcm9idXN0LCBtX25vdHJvYnVzdCwgaW50ZXJjZXB0ID0gRkFMU0UpICsgdGhlbWVfYncoKQ0KYGBgDQoNCldlIGRvIG5vdCBzZWUgZHJhbWF0aWNhbGx5IGRpZmZlcmVudCByZXN1bHRzIGJldHdlZW4gb3VyIHJvYnVzdCBhbmQgbm9uLXJvYnVzdCANCmVzdGltYXRlcy4gWW91IGNhbiBjb25maXJtIHRoaXMgYnkgY29tcGFyaW5nIHRoZSBjb2VmZmljaWVudHMgYW5kIHN0YW5kYXJkIA0KZXJyb3JzIGRpcmVjdGx5IHVzaW5nIHRoZSBzdW1tYXJ5IGZ1bmN0aW9uIGZvciBlYWNoIG1vZGVsLiANCg0KYGBge3J9DQpzdW1tYXJ5KG1fbm90cm9idXN0KQ0Kc3VtbWFyeShtX3JvYnVzdCkNCmBgYA0KDQoNCk9uZSBvZiB0aGUgcHJvYmxlbXMgd2Ugd2VyZSBsb29raW5nIHRvIHNvbHZlIGlzIHRoZSBkaXN0cmlidXRpb24gb2Ygb3VyIGVycm9ycy4NCldlIGNhbiBydW4gYSBtb2RlbCBjaGVjayB0byBzZWUgaWYgdGhlIHJlc2lkdWFscyBmcm9tIG91ciByb2J1c3QgbW9kZWwgYXJlIGJldHRlci4gDQoNCmBgYHtyfQ0KcGxvdGRmIDwtIGRhdGEuZnJhbWUoeSA9IG1fcm9idXN0JG1vZGVsJHNzMiwgDQogICAgICAgICAgICAgICAgICAgICBwcmVkX3JvYnVzdCA9IHByZWRpY3QobV9yb2J1c3QsIGRhdGEgPSBtX3JvYnVzdCRtb2RlbCksIA0KICAgICAgICAgICAgICAgICAgICAgcHJlZF9ub3JvYnVzdCA9IHByZWRpY3QobV9ub3Ryb2J1c3QsIGRhdGEgPSBtX3JvYnVzdCRtb2RlbCkpDQpwbG90ZGYkcmVzaWRfcm9idXN0IDwtIHBsb3RkZiR5IC0gcGxvdGRmJHByZWRfcm9idXN0DQpwbG90ZGYkcmVzaWRfbm9yb2J1c3QgPC0gcGxvdGRmJHkgLSBwbG90ZGYkcHJlZF9ub3JvYnVzdA0KDQpnZ3Bsb3QocGxvdGRmKSArIA0KICBnZW9tX3BvaW50KGFscGhhID0gSSgwLjEpLCBhZXMoeCA9IHByZWRfbm9yb2J1c3QsIHkgPSByZXNpZF9ub3JvYnVzdCkpICsgDQogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gMikgKw0KICBnZW9tX3Ntb290aChhZXMoeCA9IHByZWRfcm9idXN0LCB5ID0gcmVzaWRfcm9idXN0KSwgDQogICAgICAgICAgICAgIGNvbG9yID0gSSgicmVkIiksIHNlID0gRkFMU0UpICsgDQogIGdlb21fc21vb3RoKGFlcyh4ID0gcHJlZF9ub3JvYnVzdCwgeSA9IHJlc2lkX25vcm9idXN0KSwgDQogICAgICAgICAgICAgIGNvbG9yID0gSSgiYmxhY2siKSwgc2U9RkFMU0UpICsgDQogIA0KICB0aGVtZV9idygpDQoNCmBgYA0KDQpUaGUgcm9idXN0IGVzdGltYXRvciBkb2VzIGFsbW9zdCBub3RoaW5nIHRvIGltcHJvdmUgb3VyIHJlc2lkdWFsIHBsb3QgZWl0aGVyIC0gDQpldmVuIGF0IGV4dHJlbWUgdmFsdWVzIHRoZSBhdHRlbnVhdGlvbiBvZiB0aGUgcmVzaWR1YWxzIGlzIHNtYWxsLiANClNvIHRoZXJlIGFyZSBsaW1pdHMgdG8gdGhlIHN0cmVuZ3RoIG9mIHJvYnVzdCByZWdyZXNzaW9uIGFpZGluZyBpbiB0aGUgcHJlc2VuY2UgDQpvZiBvdXRsaWVycy4NCg0KSG93ZXZlciwgdGhlIHJvYnVzdCBlc3RpbWF0ZXMgb2Ygb3VyIHBhcmFtZXRlcnMsIGNvZWZmaWNpZW50cywgc2hvdWxkIGJlIGxlc3MNCmluZmx1ZW5jZWQgYnkgdGhlc2Ugb3V0bGllcnMuIFNvIHJvYnVzdG5lc3MgZG9lcyBub3QgaGVscCB1cyB3aXRoIHByZWRpY3Rpb24NCnZlcnkgbXVjaCwgYnV0IGl0IGRvZXMgaGVscCB1cyBhZGp1c3QgZm9yIHRoZSBiaWFzIHRoYXQgY2FuIG9jY3VyIGluIG91cg0KY29lZmZpY2llbnRzIHdoZW4gdGhlcmUgYXJlIHN0cm9uZyBvdXRsaWVycyBpbiB0aGUgZGF0YS4gVGhpcyBpcyB3aHkgaXQgaXMNCmltcG9ydGFudCB0byB1bmRlcnN0YW5kIHRoZSBwdXJwb3NlIG9mIHRoZSBtb2RlbHMgd2UgYXJlIGZpdHRpbmcsIGluIG9yZGVyIHRvDQp1bmRlcnN0YW5kIHdoZW4gZGlmZmVyZW50IGNvcnJlY3Rpb25zIGFyZSBhcHByb3ByaWF0ZS4NCldlIHNob3VsZCBwcmVmZXIgdGhlIHJvYnVzdCBjb2VmZmljaWVudHMgZm9yIHVuZGVyc3RhbmRpbmcgdGhlIHJlbGF0aW9uc2hpcHMNCndpdGhpbiBvdXIgZGF0YSwgYnV0IHdlIG1heSBub3QgcHJlZmVyIHRoZSByb2J1c3QgY29lZmZpY2llbnRzIGZvciB0aGUgcHVycG9zZXMNCm9mIG1ha2luZyBuZXcgcHJlZGljdGlvbnMgb24gZnV0dXJlIGRhdGEuDQoNCiMjIyMgUm9idXN0IFN0YW5kYXJkIEVycm9ycw0KDQpJbiBhZGRpdGlvbiB0byByb2J1c3QgcmVncmVzc2lvbiBlc3RpbWF0ZXMsIGEgbW9yZSBzdGFuZGFyZCBwcm9jZWR1cmUgaXMgdG8NCmFkanVzdCB0aGUgc3RhbmRhcmQgZXJyb3JzIGluIG91ciBtb2RlbCB3aXRoIGEgcG9zdGVzdGltYXRpb24gYWRqdXN0bWVudC4gSW4NCnRoaXMgY2FzZSwgdGhlIG1vZGVsIGNvZWZmaWNpZW50cyBzdGF5IHRoZSBzYW1lIC0gdGhleSBhcmUgdGhlIE9MUyBiZXRhcyAtIGJ1dA0KdGhlIHN0YW5kYXJkIGVycm9ycyBoYXZlIGJlZW4gYWRqdXN0ZWQgdG8gcmVmbGVjdCB0aGUgbm9uLWluZGVwZWRlbmNlIGJldHdlZW4NCm91ciBvYnNlcnZhdGlvbnMuDQoNCkluIFIgdGhpcyBwcm9jZWR1cmUgaXMgbXVjaCBtb3JlIGNvbXBsaWNhdGVkIHRoYW4gU3RhdGEsIG93aW5nIHRvIFIncyByZWZ1c2FsIA0KdG8gaW1wbGVtZW50IGRlZmF1bHRzLiBUaGVyZSBhcmUgbWFueSBkaWZmZXJlbnQgYWNjZXB0YWJsZSBlc3RpbWF0b3JzIHRvIGFkanVzdCANCnRoZSBzdGFuZGFyZCBlcnJvcnMgLSBTdGF0YSB1c2VzIGEgZGVmYXVsdCBhbmQgZG9lcyBub3QgYm90aGVyIHRoZSB1c2VyLiBSIA0KcmVmdXNlcy4gVG8gcmVwbGljYXRlIHRoZSBiZWhhdmlvciBvZiBTdGF0YSwgdXNlICJIQzEiIGFzIHRoZSBhZGp1c3RtZW50LiANCg0KYGBge3J9DQpsaWJyYXJ5KHNhbmR3aWNoKQ0KbGlicmFyeShsbXRlc3QpDQpzdW1tYXJ5KG1fbm90cm9idXN0KSRjb2VmZmljaWVudHMNCmNvZWZ0ZXN0KG1fbm90cm9idXN0LCANCiAgICAgICAgIHZjb3YgPSB2Y292SEMobV9ub3Ryb2J1c3QsICJIQzEiKSkNCmBgYA0KDQpSZW1lbWJlciwgdGhpcyBhZGp1c3RtZW50IG9ubHkgYWZmZWN0cyB0aGUgc3RhbmRhcmQgZXJyb3JzIC0gdGhlIGNvZWZmaWNpZW50cyANCnJlbWFpbiB0aGUgc2FtZS4gKElmIHRoaXMgY29kZSBkb2Vzbid0IHdvcmssIG1ha2Ugc3VyZSB0byBpbnN0YWxsIA0KYGdyaWRFeHRyYWAgZmlyc3QpDQoNCmBgYHtyfQ0KZ3JpZEV4dHJhOjpncmlkLmFycmFuZ2UoY29lZnBsb3QobV9ub3Ryb2J1c3QsIGludGVyY2VwdCA9IEZBTFNFLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNvcnQgPSAiYWxwaGFiZXRpY2FsIikgKyB0aGVtZV9idygpLCANCiAgICAgICAgICAgICAgICAgICAgICAgIGNvZWZwbG90X2NvZWZ0ZXN0KA0KICAgICAgICAgICAgICAgICAgICAgICAgICBjb2VmdGVzdChtX25vdHJvYnVzdCwgdmNvdiA9IHZjb3ZIQyhtX25vdHJvYnVzdCwgIkhDMSIpKSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgIGludGVyY2VwdCA9IEZBTFNFKQ0KICAgICAgICAgICAgICAgICAgICAgICAgKQ0KYGBgDQoNCkluIHRoaXMgY2FzZSwgdGhlIGFkanVzdG1lbnQgbG9va3MgbWlub3IgLSBpdCdzIG5lYXJseSBpbXBvc3NpYmxlIHRvIGRpc3Rpbmd1aXNoIA0KaW4gb3VyIGNvZWZmaWNpZW50IHBsb3RzIGFuZCBkb2VzIG5vdCBhZmZlY3Qgb3VyIGRlY2lzaW9uIGFib3V0IHRoZSBzdGF0aXN0aWNhbCANCnNpZ25pZmljYW5jZSBvZiBhbnkgb2Ygb3VyIHByZWRpY3RvcnMuIFRoaXMgaXMgYSBnb29kIHNpZ24hDQoNCiMjIyMgQ2x1c3RlciBTdGFuZGFyZCBFcnJvcnMNCg0KVGhlcmUgaXMgYSBzcGVjaWFsIGNhc2Ugb2Ygcm9idXN0IHN0YW5kYXJkIGVycm9ycyB0aGF0IGFyZSB2ZXJ5IGltcG9ydGFudCBpbg0KZWR1Y2F0aW9uIHJlZ3Jlc3Npb24gbW9kZWxpbmcgLSBjbHVzdGVyIHN0YW5kYXJkIGVycm9ycy4gQ2x1c3RlciBzdGFuZGFyZCBlcnJvcnMNCmFsbG93IHVzIHRvIGFjY291bnQgZm9yIHRoZSBmYWN0IHRoYXQgb2JzZXJ2YXRpb25zIGRyYXduIGZyb20gdGhlIHNhbWUgImNsdXN0ZXIiDQpjb3Zhcnkgd2l0aCBlYWNoIG90aGVyLCBhbmQgdGh1cyB0aGV5IGFyZSBub3QgZnVsbHkgaW5kZXBlbmRlbnQuIFdlIGNhbiB1c2UgdGhpcw0KaW5mb3JtYXRpb24gdG8gZ2V0IGEgbW9yZSBhY2N1cmF0ZSBlc3RpbWF0ZSBvZiBvdXIgbW9kZWwncyBjZXJ0YWludHkgLSBnZW5lcmFsbHkNCmJ5IGluZmxhdGluZyB0aGUgc3RhbmRhcmQgZXJyb3JzIGEgYml0IHRvIGFjY291bnQgZm9yIHRoaXMgZHVwbGljYXRlZA0KaW5mb3JtYXRpb24uDQoNClRoaXMgdGVjaG5pcXVlIGlzIGltcG9ydGFudCBpbiBlZHVjYXRpb24gYmVjYXVzZSBlZHVjYXRpb24gZGF0YSBpcyBmdWxsIG9mDQoiY2x1c3RlcnMiIC0gcmVwZWF0ZWQgdGVzdCBzY29yZXMgZm9yIHRoZSBzYW1lIHN0dWRlbnQsIHN0dWRlbnRzIGluIHRoZSBzYW1lDQpjbGFzc3Jvb20sIGNsYXNzcm9vbXMgaW4gdGhlIHNhbWUgc2Nob29sLCBzY2hvb2xzIGluIHRoZSBzYW1lIGRpc3RyaWN0LCBhbmQgZXZlbg0KZGlzdHJpY3RzIGluIHRoZSBzYW1lIHN0YXRlLg0KDQpMZXQncyBqdXN0IGxvb2sgYXQgb25lIGV4YW1wbGUgd2hlcmUgd2UgY2x1c3RlciBvbiB0aGUgc2Nob29sIGRpc3RyaWN0IGNvZGUgaW4NCm91ciBkYXRhLg0KDQpgYGB7cn0NCmNsdXN0ZXJfdmNvdiA8LSBnZXRfQ0xfdmNvdihtX25vdHJvYnVzdCwgZnVsbF9kYXRhJGRpc3RpZCkNCnN1bW1hcnkobV9ub3Ryb2J1c3QpJGNvZWZmaWNpZW50cw0KY29lZnRlc3QobV9ub3Ryb2J1c3QsIHZjb3YgPSBjbHVzdGVyX3Zjb3YpDQoNCmNvZWZwbG90KG1fbm90cm9idXN0LCBpbnRlcmNlcHQgPSBGQUxTRSkNCmNvZWZwbG90X2NvZWZ0ZXN0KGNvZWZ0ZXN0KG1fbm90cm9idXN0LCB2Y292ID0gY2x1c3Rlcl92Y292KSwgaW50ZXJjZXB0ID0gRkFMU0UpDQoNCmBgYA0KDQpXZSBzZWUgdGhhdCBjbHVzdGVyaW5nIGhhcyBhIG11Y2ggYmlnZ2VyIGltcGFjdCBvbiBvdXIgc3RhbmRhcmQgZXJyb3JzIHRoYW4gdGhlDQpyb2J1c3RuZXNzIGNvcnJlY3Rpb24gd2UgYXBwbGllZCBhYm92ZS4gSGVyZSB3ZSBzZWUgdGhlIHN0YW5kYXJkIGVycm9ycyBpbmNyZWFzZQ0KYnkgMi01eCBpbiBzaXplLCByZXN1bHRpbmcgaW4gdGhlIGBhdHRfcmF0ZWAgcHJlZGljdG9yIHRoYXQgb25jZSBzZWVtZWQNCnN0YXRpc3RpY2FsbHkgc2lnbmlmaWNhbnQgdG8gbm8gbG9uZ2VyIGJlIHNvLg0KDQpDbHVzdGVyaW5nIGluIHRoaXMgd2F5IGRvZXMgbm90IGhlbHAgd2l0aCBvdXIgcmVzaWR1YWxzIHRob3VnaCBiZWNhdXNlIHdoaWxlIGl0DQpjaGFuZ2VzIHRoZSBzdGFuZGFyZCBlcnJvcnMgYXJvdW5kIG91ciBjb2VmZmljaWVudHMgYW5kIHByZWRpY3Rpb25zLCBpdCBkb2VzIG5vdA0KY2hhbmdlIHRoZSBjb2VmZmljaWVudHMgd2hpY2ggYXJlIHVzZWQgdG8gZ2VuZXJhdGUgdGhlIHByZWRpY3Rpb24uIFNlZSBmb3INCnlvdXJzZWxmOg0KDQpgYGB7cn0NCnBsb3RkZiA8LSBkYXRhLmZyYW1lKHkgPSBmdWxsX2RhdGEkc3MyLCANCiAgICAgICAgICAgICAgICAgICAgIHloYXRfbnIgPSBmaXR0ZWQobV9ub3Ryb2J1c3QpLCANCiAgICAgICAgICAgICAgICAgICAgIHloYXRfciA9IHByZWRpY3Qucm9idXN0KG1fbm90cm9idXN0LCBkYXRhID0gZnVsbF9kYXRhLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHJvYnVzdF92Y292ID0gY2x1c3Rlcl92Y292KSkNCg0KcGxvdGRmJHJlc2lkX3JvYnVzdCA8LSBwbG90ZGYkeSAtIHBsb3RkZiR5aGF0X3IuZml0LmZpdA0KcGxvdGRmJHJlc2lkX25vcm9idXN0IDwtIHBsb3RkZiR5IC0gcGxvdGRmJHloYXRfbnINCg0KZ2dwbG90KHBsb3RkZikgKyANCiAgZ2VvbV9wb2ludChhbHBoYSA9IEkoMC4xKSwgYWVzKHggPSB5aGF0X25yLCB5ID0gcmVzaWRfbm9yb2J1c3QpKSArIA0KICBnZW9tX3Ntb290aChhZXMoeCA9IHloYXRfci5maXQuZml0LCB5ID0gcmVzaWRfcm9idXN0KSwgDQogICAgICAgICAgICAgIGNvbG9yID0gSSgicmVkIiksIHNlID0gRkFMU0UpICsgDQogIGdlb21fc21vb3RoKGFlcyh4ID0geWhhdF9uciwgeSA9IHJlc2lkX25vcm9idXN0KSwgDQogICAgICAgICAgICAgIGNvbG9yID0gSSgiYmxhY2siKSwgc2U9RkFMU0UpICsgDQogIHRoZW1lX2J3KCkNCg0KDQpgYGANCg0KDQoNCmBgYHtyLCBldmFsID0gRkFMU0V9DQpjb2VmdGVzdChtXzQsIHZjb3YgPSB2Y292SEMobV80LCAiSEMxIikpDQpjb2VmdGVzdChtXzQsIHZjb3YgPSBzYW5kd2ljaCkNCmNvZWZ0ZXN0KG1fNCwgdmNvdiA9IHZjb3ZIQyhtXzQsICJIQzAiKSkNCg0KDQojIGNoZWNrIHRoYXQgInNhbmR3aWNoIiByZXR1cm5zIEhDMA0KY29lZnRlc3QobG1BUEksIHZjb3YgPSBzYW5kd2ljaCkgICAgICAgICAgICAgICAgIyByb2J1c3Q7IHNhbmR3aWNoDQpjb2VmdGVzdChsbUFQSSwgdmNvdiA9IHZjb3ZIQyhsbUFQSSwgIkhDMCIpKSAgICAjIHJvYnVzdDsgSEMwIA0KDQojIGNoZWNrIHRoYXQgdGhlIGRlZmF1bHQgcm9idXN0IHZhci1jb3YgbWF0cml4IGlzIEhDMw0KY29lZnRlc3QobG1BUEksIHZjb3YgPSB2Y292SEMobG1BUEkpKSAgICAgICAgICAgIyByb2J1c3Q7IEhDMyANCmNvZWZ0ZXN0KGxtQVBJLCB2Y292ID0gdmNvdkhDKGxtQVBJLCAiSEMzIikpICAgICMgcm9idXN0OyBIQzMgKGRlZmF1bHQpDQoNCiMgcmVwcm9kdWNlIHRoZSBTdGF0YSBkZWZhdWx0DQpjb2VmdGVzdChsbUFQSSwgdmNvdiA9IHZjb3ZIQyhsbUFQSSwgIkhDMSIpKSAgICAjIHJvYnVzdDsgSEMxIChTdGF0YSBkZWZhdWx0KQ0KDQpgYGANCg0KDQojIyBEZXNjcmlwdGl2ZSBSZWdyZXNzaW9uDQoNCkFzIHdlIGRpc2N1c3NlZCBpbiB0aGUgbGVjdHVyZSwgYSBwb3dlcmZ1bCB1c2Ugb2YgcmVncmVzc2lvbiBpcyB0byBjYWxjdWxhdGUgYW4gDQplc3RpbWF0ZSBvZiBhIHZhcmlhYmxlIGFjY291bnRpbmcgZm9yIG11bHRpcGxlIG90aGVyIGZhY3RvcnMgc2ltdWx0YW5lb3VzbHkuIA0KUmVncmVzc2lvbiBhbGxvd3MgdXMgdG8gbG9vayBhdCBhIGtleSB2YXJpYWJsZSBsaWtlIGEgcG9zdHRlc3Qgc2NvcmUgYWZ0ZXIgDQphY2NvdW50aW5nIGZvciBrZXkgYXR0cmlidXRlcyBvZiB0aGUgdGVzdCB0YWtlcnMsIGxpa2UgdGhlaXIgZGVtb2dyYXBoaWMgDQpjaGFyYWN0ZXJpc3RpY3Mgb3IgdGhlaXIgcHJpb3IgYXR0ZW5kYW5jZS4gDQoNCg0KIyMjIERlbW9ncmFwaGljIEFkanVzdG1lbnRzDQoNCk9uZSBvZiB0aGUgdGhpbmdzIHBlb3BsZSBsaWtlIHRvIGRvIGlzIGFkanVzdCBmb3IgbXVsdGlwbGUgc3R1ZGVudCBvciANCnNjaG9vbCBjb25kaXRpb25zIHNpbXVsdGFuZW91c2x5IGFuZCB0aGVuIGlkZW50aWZ5IHBlcmZvcm1hbmNlLiBXZSBrbm93IA0KYWxsIHRoZSBwYXJ0cyBuZWNlc3NhcnkgdG8gZG8gdGhpcyAtIHdlIGRlZmluZSBhbmQgZml0IGEgbW9kZWwsIGdldCBpdHMgDQpwcmVkaWN0aW9ucywgYW5kIHRoZW4gd2UgcGxvdCB0aGUgcHJlZGljdGlvbnMgdXNpbmcgRURBIHRlY2huaXF1ZXMgd2UgaGF2ZSANCmFscmVhZHkgbWFzdGVyZWQuIA0KDQpgYGB7cn0NCiMgTWlzc2luZyBkYXRhIHJlc3VsdHMgZnJvbSBkaXZpZGluZyBieSAwLCBzbyB3ZSBjYW4gc2V0IHRoZXNlIGF0IDANCmZ1bGxfZGF0YSRlbGxfcGVyW2lzLm5hKGZ1bGxfZGF0YSRlbGxfcGVyKV0gPC0gMA0KZnVsbF9kYXRhJHN3ZF9wZXJbaXMubmEoZnVsbF9kYXRhJHN3ZF9wZXIpXSA8LSAwDQojIA0KIyBGaXQgYWRqdXN0bWVudCBtb2RlbA0KYWRqX21vZGVsIDwtIGxtKHNzMiB+IGZhY3RvcihncmFkZSkgKiBmYWN0b3IodGVzdF95ZWFyKSArIHN1YmplY3QgKyBmcmxfcGVyICsgDQogICAgICAgICAgICAgICAgICBlbGxfcGVyICsgc3dkX3BlciArIG4xICsgbG9jYWxlX2Rlc2MsIGRhdGEgPSBmdWxsX2RhdGEpDQpmdWxsX2RhdGEkYWRqX3NzMiA8LSBwcmVkaWN0KGFkal9tb2RlbCkNCg0KDQpgYGANCg0KDQpUaGlzIGNvZGUganVzdCBzZWxlY3RzIHNvbWUgc2FtcGxlIGRpc3RyaWN0cyBhbmQgY3JlYXRlcyB0d28gYm94cGxvdHMgb2YgdGhlaXIgDQpzY2hvb2wvZ3JhZGUgY29ob3J0IHNjb3JlcyBiZWZvcmUgYW5kIGFmdGVyIGFkanVzdGluZyBmb3IgZGVtb2dyYXBoaWNzIC0gYXMgDQp3ZSBsb29rZWQgYXQgaW4gdGhlIGxlY3R1cmUuIA0KDQpgYGB7cn0NCmV4YW1wbGVfZGlzdHJpY3RzIDwtIGMoMTU1LCAzNTIsIDI3NywgOTgsIDMyMywgNzUsIDQ0LCAxMzEsIDI4MCwgMjA1KQ0KcDEgPC0gZ2dwbG90KGZ1bGxfZGF0YVtmdWxsX2RhdGEkZGlzdGlkICVpbiUgZXhhbXBsZV9kaXN0cmljdHMsIF0sIA0KICAgICAgIGFlcyh5ID0gYWRqX3NzMiwgeCA9IGZhY3RvcihkaXN0aWQpKSkgKyANCiAgZ2VvbV9ib3hwbG90KCkgKyBnZW9tX2ppdHRlcigpICsgdGhlbWVfYncoKSArIA0KICBsYWJzKHRpdGxlID0gIkFkanVzdGVkIFNjb3JlcyIsIHggPSAiRElTVElEIiwgeSA9ICJBZGp1c3RlZCBTUzIiKSArIA0KICB5bGltKDEwMDAsIDEzNTApDQoNCnAyIDwtIGdncGxvdChmdWxsX2RhdGFbZnVsbF9kYXRhJGRpc3RpZCAlaW4lIGV4YW1wbGVfZGlzdHJpY3RzLCBdLCANCiAgICAgICBhZXMoeSA9IHNzMiwgeCA9IGZhY3RvcihkaXN0aWQpKSkgKyANCiAgZ2VvbV9ib3hwbG90KCkgKyBnZW9tX2ppdHRlcigpICsgdGhlbWVfYncoKSArIA0KICBsYWJzKHRpdGxlID0gIlVuYWRqdXN0ZWQgU2NvcmVzIiwgeCA9ICJESVNUSUQiLCB5PSAiT2JzZXJ2ZWQgU1MyIikgKyANCiAgeWxpbSgxMDAwLCAxMzUwKSANCg0KDQpncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwMSwgcDIsIG5jb2wgPSAyKQ0KYGBgDQoNCg0KV2UgbWF5IGFsc28gd2FudCB0byBlc3RpbWF0ZSB0aGUgaW5mbHVlbmNlIG9mIG9uZSB2YXJpYWJsZSBvbiBhbm90aGVyLCBidXQgDQphY2NvdW50IGZvciB0aGVzZSBjb25kaXRpb25hbCBkaWZmZXJlbmNlcy4gT25lIGNvbW1vbiBxdWVzdGlvbiAoaC90IHRvIEFiaWdhaWwpIA0KaXMgdGhlIHJlbGF0aW9uc2hpcCBiZXR3ZWVuIGF0dGVuZGFuY2UgYW5kIHRlc3Qgc2NvcmVzLiBXZSBvZnRlbiB3YW50IHRvIGlkZW50aWZ5IA0KYSBjcml0aWNhbCB0aHJlaG9sZCBpbiB0aGlzIHJlbGF0aW9uc2hpcC4gSW4gdGhlIHBsb3QgYmVsb3csIHdlIHVzZSBhIHNjYXR0ZXJwbG90IA0Kc21vb3RoZXIgYmV0d2VlbiB0aGUgcG9zdHRlc3Qgc2NvcmUgYW5kIHRoZSBzY2hvb2wgYXR0ZW5kYW5jZSByYXRlIHRvIGlkZW50aWZ5IA0Kd2hlcmUgdGhlIHJlbGF0aW9uc2hpcCBiZXR3ZWVuIGF0dGVuZGFuY2UgYW5kIHRlc3Qgc2NvcmVzIGNoYW5nZXMuIFdlIGV4cGVjdCANCnRoZXJlIHRvIGJlIGRpbWluaXNoaW5nIHJldHVybnMgYXQgYm90aCBlbmRzIC0tIGFkZGl0aW9uYWwgZGF5cyBhdHRlbmRpbmcgZG8gbm90IA0KeWllbGQgaGlnaGVyIHRlc3Qgc2NvcmVzIGF0IHNvbWUgcG9pbnQsIHRoZSBzYW1lIHdheSBhZGRpdGlvbmFsIGFic2VuY2VzIHN0b3AgDQpsb3dlcmluZyB0ZXN0IHNjb3JlcyBhdCBzb21lIHBvaW50LiANCg0KQnV0LCB3ZSBtYXkgYmUgaW50ZXJlc3RlZCBpbiB1bmRlcnN0YW5kaW5nIGlmIHRoaXMgcmVsYXRpb25zaGlwIGlzIGRpZmZlcmVudCANCmFmdGVyIHdlIGhhdmUgdGFrZW4gb3RoZXIgZmFjdG9ycyBhYm91dCBzY2hvb2xzIG9yIHN0dWRlbnRzIGludG8gYWNjb3VudCAtIHNvIA0KbGV0J3MgY29tcGFyZSBvdXIgc2NvcmVzIGFkanVzdGVkIGZvciBzY2hvb2wgY29tcG9zaXRpb24gZWZmZWN0cyBhZ2FpbnN0IHRoZSANCnVuYWRqdXN0ZWQgc2NvcmVzLCBhbmQgbG9vayBhdCBob3cgdGhpcyByZWxhdGlvbnNoaXAgY2hhbmdlcy4gDQoNCmBgYHtyfQ0KDQpwMSA8LSBnZ3Bsb3QoZnVsbF9kYXRhW2Z1bGxfZGF0YSRhdHRfcmF0ZSA+IDUwLF0sIA0KICAgICAgIGFlcyh5ID0gYWRqX3NzMiwgeCA9IGF0dF9yYXRlKSkgKyANCiAgICBnZW9tX2ppdHRlcihhbHBoYT0xLzcpICsgdGhlbWVfYncoKSArIA0KICAgIGxhYnModGl0bGUgPSAiQWRqdXN0ZWQgU2NvcmVzIiwgeCA9ICJQcmUtVGVzdCIsIHk9ICJBZGp1c3RlZCBTUzIiKSArIA0KICAgIGdlb21fc21vb3RoKHNlPUZBTFNFKSAgKyB5bGltKDEwMDAsIDEzNTApIA0KDQpwMiA8LSAgZ2dwbG90KGZ1bGxfZGF0YVtmdWxsX2RhdGEkYXR0X3JhdGUgPiA1MCxdLCANCiAgICAgICBhZXMoeSA9IHNzMiwgeCA9IGF0dF9yYXRlKSkgKyAgDQogIGdlb21faml0dGVyKGFscGhhID0gMS83KSArIHRoZW1lX2J3KCkgKyANCiAgbGFicyh0aXRsZSA9ICJVbmFkanVzdGVkIFNjb3JlcyIsIHggPSAiQXR0ZW5kYW5jZSIsIHk9ICJPYnNlcnZlZCBTUzIiKSArIA0KICBnZW9tX3Ntb290aChzZT1GQUxTRSkgKyB5bGltKDEwMDAsIDEzNTApIA0KDQpncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwMSwgcDIsIG5jb2wgPSAyKQ0KDQpgYGANCg0KVGhpcyBwbG90IGRvZXMgbm90IGRpc3BsYXkgbXVjaCBkaWZmZXJlbmNlLCBidXQgaXQgaXMgc2VlbXMgY2xlYXIgdGhhdCBpbiB0aGUgDQphZGp1c3RlZCBzY29yZXMsIHRoZSByZWxhdGlvbnNoaXAgd2l0aCB0aGUgcG9zdHRlc3QgbGV2ZWxzIG9mZiBhdCBzbGlnaHRseSANCmhpZ2hlciBhdHRlbmRhbmNlIHJhdGVzIHRoYW4gdGhlIHVuYWRqdXN0ZWQgc2NvcmVzLiBUaGlzIHN1Z2dlc3RzIHRoYXQgYWZ0ZXIgDQphY2NvdW50aW5nIGZvciBkZW1vZ3JhcGhpY3MsIHRoZSBjdXRwb2ludCB3aGVyZSBhdHRlbmRhbmNlIG1hdHRlcnMgbGVzcyBpcyANCmhpZ2hlciwgYW5kIHdlIG1heSB3YW50IHRvIGNvbnNpZGVyIHJlY29tbWVuZGluZyBhIGhpZ2hlciBhdHRlbmRhbmNlIHRocmVzaG9sZCANCnRoYW4gd2Ugd291bGQgaWYgd2Ugb25seSBjb25zaWRlcmVkIHRoZSB1bmFkanVzdGVkIHNjb3Jlcy4gDQoNCiMjIyBTaW1wbGlmaWVkIFZhbHVlLUFkZGVkDQoNClZhbHVlLWFkZGVkIGlzIGFub3RoZXIgZGVzY3JpcHRpdmUgdXNlIG9mIHJlZ3Jlc3Npb24gd2hlcmUgd2UgZml0IGEgbW9kZWwgdGhhdCANCndlIGhvcGUgZXhwbGFpbnMgYWxsIHRoZSBmYWN0b3JzIGNvbnRyaWJ1dGluZyB0byBzdHVkZW50IGdyb3d0aCB0aGF0IGFyZSANCm9ic2VydmFibGUsIGFuZCB0aGVuIHdlIGxvb2sgYXQgdGhlIHJlc2lkdWFsIGVycm9yIGZyb20gdGhhdCBtb2RlbCwgdGhlIA0KdW5vYnNlcnZlZCBlcnJvciwgYW5kIGludGVycHJldCBpdCBhcyBhdHRyaWJ1dGFibGUgdG8gdGhlIHNjaG9vbCBvciB0ZWFjaGVyIA0KZXRjLiANCg0KYGBge3J9DQoNCnZhbV9tb2RlbCA8LSBsbShzczIgfiBzczEgKyBmYWN0b3IoZ3JhZGUpICogZmFjdG9yKHRlc3RfeWVhcikgKyBmcmxfcGVyICsgDQogICAgICAgICAgICAgICAgICBlbGxfcGVyICsgc3dkX3BlciArIG4xICsgbG9jYWxlX2Rlc2MgKyBhdHRfcmF0ZSArIHN1YmplY3QgKw0KICAgICAgICAgICAgICAgICAgZmFjdG9yKGRpc3RpZCksIGRhdGEgPSBmdWxsX2RhdGEpDQpmdWxsX2RhdGEkdmFtX3NzMiA8LSBwcmVkaWN0KHZhbV9tb2RlbCkNCmZ1bGxfZGF0YSR2YW1fc2NvcmUgPC0gcnN0YW5kYXJkKHZhbV9tb2RlbCkNCg0KDQpnZ3Bsb3QoZnVsbF9kYXRhW2Z1bGxfZGF0YSRkaXN0aWQgJWluJSBleGFtcGxlX2Rpc3RyaWN0cywgXSwgDQogICAgICAgYWVzKHkgPSB2YW1fc2NvcmUsIHggPSBmYWN0b3IoZGlzdGlkKSkpICsgDQogIGdlb21fYm94cGxvdChjb2xvcj0iZ3JheTgwIikgKyBnZW9tX2ppdHRlcigpICsgdGhlbWVfYncoKSArIA0KICBsYWJzKHRpdGxlID0gIlZhbSBTY29yZXMiLCB4ID0gIkRJU1RJRCIsIHk9ICJPYnNlcnZlZCBTUzIiKQ0KDQoNCmBgYA0KDQoNClZBTSBtb2RlbHMgYXJlIG9mdGVuIG1vcmUgY29tcGxleCBhbmQgaW5jbHVkZSBwb3N0LWVzdGltYXRpb24gYWRqdXN0bWVudHMgb2YgdGhlDQpyZXNpZHVhbHMgdG8gYWNjb3VudCBmb3IgdGhlIHNhbXBsZSBzaXplIG9yIGFnZ3JlZ2F0aW9uIG9mIHNjaG9vbHMgaW50bw0KZGlzdHJpY3RzIG9yIGNsYXNzcm9vbXMgaW50byBzY2hvb2xzLiBCdXQsIHRoaXMgaXMgdGhlIGJhc2ljIGZvcm1hdCBvZg0KdmFsdWUtYWRkZWQgbW9kZWxpbmcgYW5kIHRoZSBpbnR1aXRpb24gdGhhdCB1bmRlcmxpZXMgaXQuDQoNCkFzIGEgZmluYWwgcXVlc3Rpb24gLSBob3cgbXVjaCBiZXR0ZXIgZG9lcyB0aGlzIHZhbHVlLWFkZGVkIG1vZGVsIGZpdCB0aGUgDQpkYXRhIHRoYW4gb3RoZXIgbW9kZWxzIHdlIGhhdmUgdXNlZCBpbiB0aGUgZ3VpZGU/IEhvdyBtaWdodCB5b3UgZXZhbHVhdGUgdGhhdD8gDQpXaGF0IGFyZSB0aGUgY29uc2VxdWVuY2VzIG9mIHRoYXQgZm9yIGludGVycHJldGF0aW9uPw0KDQojIyBCT05VUzogVGVzdGluZyBNYW55IE1vZGVscw0KDQpBYm92ZSwgd2UgaGF2ZSBzaG93biByZWdyZXNzaW9uIGRpYWdub3N0aWNzIGZvciBqdXN0IG9uZSBtb2RlbC4gQnV0LCBub3cgb3VyIA0KY29sbGVhZ3VlIGhhcyBwcm9wb3NlZCBmaWZ0eS4gVGhlIG1vZGVsIGFib3ZlIHNob3dzIHNpZ25zIG9mIG1pc3NwZWNpZmljYXRpb24gYW5kIA0KaGV0ZXJvc2tlZGFzdGljaXR5IC0gd2hpY2ggZ2l2ZXMgdXMgcmVhc29uIHRvIGNvbnNpZGVyIHRlc3RpbmcgdGhlIG90aGVyIDQ5IA0KbW9kZWxzIGFuZCBjb21wYXJpbmcgdGhlbS4gDQoNClRvIGRvIHRoaXMsIHdlIHdpbGwgdXNlIGFuIFIgZnVuY3Rpb24gdGhhdCBhcHBsaWVzIHRoZSB0ZXN0IHRvIGVhY2ggbW9kZWwgDQphbmQgcmV0dXJucyBhIGRhdGEuZnJhbWUgd2l0aCBvbmUgcm93IHBlciBtb2RlbCBhbmQgb25lIGNvbHVtbiBwZXIgdGVzdC4gSXQncyANCk9LIGlmIHlvdSBkb24ndCBmZWVsIGNvbWZvcnRhYmxlIHdyaXRpbmcgYW5kIGVkaXRpbmcgZnVuY3Rpb25zIGp1c3QgeWV0IC0gYnV0IA0KdGhpcyBpcyBhIHVzZWZ1bCBleGFtcGxlIG9mIGhvdyB0aGUgZmxleGliaWxpdHkgdG8gd3JpdGluZyBmdW5jdGlvbnMgY2FuIGFsbG93IHlvdSANCnRvIGRvIHNjYWxlIHlvdXIgYW5hbHlzaXMgZnJvbSBvbmUgdG8gbWFueSBtb2RlbHMgZWFzaWx5LiANCg0KRmlyc3QgdGhlIGZ1bmN0aW9uLCB3ZSdsbCBjYWxsIGl0IGByZWdyZXNzX2JhdHRlcnlgIC0gYXMgaW4gYSBiYXR0ZXJ5IG9mIHJlZ3Jlc3Npb24gDQp0ZXN0cy4gDQoNCg0KYGBge3J9DQojIEZ1bmN0aW9ucyBhcmUganVzdCBuYW1lZCBvYmplY3RzIGluIFIgb2YgYSBzcGVjaWFsIHR5cGUNCiMgSGVyZSB3ZSBuYW1lIG91ciBmdW5jdGlvbiwgYW5kIHdlIHNheSB0aGF0IGl0IGlzIGEgZnVuY3Rpb24gd2l0aCB0d28gImFyZ3VtZW50cyINCiMgdGhlIGZpcnN0IG1vZGVsLCB0aGUgc2Vjb25kIG9yZGVyX3Zhci4gVGhlIGRlZmF1bHQgdmFsdWUgb2Ygb3JkZXJfdmFyIGlzIE5VTEwNCnJlZ3Jlc3NfYmF0dGVyeSA8LSBmdW5jdGlvbihtb2RlbCwgb3JkZXJfdmFyID0gTlVMTCkgew0KICAjIG1vZGVsID0gYSByZWdyZXNzaW9uIG1vZGVsIG9iamVjdCwgb2YgY2xhc3MgbG0NCiAgIyBvcmRlcl92YXIgPSBhIHZlY3RvciBvZiB2YWx1ZXMgdG8gcmVvcmRlciB0aGUgcmVncmVzc2lvbiB0ZXN0IGJ5DQogIA0KICAjIEZpcnN0LCBleHRyYWN0IHRoZSByZXNpZHVhbHMgZnJvbSB0aGUgbW9kZWwsIHdlIG5lZWQgdGhlc2UgZm9yIHNvbWUgDQogICMgdHlwZXMgb2YgcmVncmVzc2lvbiB0ZXN0cw0KICAgIG9yZGVyZWRfcmVzaWRzIDwtIHJlc2lkdWFscyhtb2RlbCkNCiAgaWYgKCFpcy5udWxsKG9yZGVyX3ZhcikpIHsgIyBpZiBvcmRlciBpcyBnaXZlbiwgcmVvcmRlciB0aGUgcmVzaWR1YWxzDQogICAgICBvcmRlcmVkX3Jlc2lkcyA8LSBvcmRlcmVkX3Jlc2lkc1tvcmRlcihvcmRlcl92YXIpXQ0KICAgICAgb3JkZXJfYnkgPC0gb3JkZXJfdmFyWzFdDQogIH0gZWxzZSB7DQogICAgIyBJZiBubyBvcmRlciBpcyBnaXZlbiwgc3BlY2lmeSB0aGF0IHRoZSBvcmRlciB2YWx1ZSBpcyBOQQ0KICAgICAgb3JkZXJfYnkgPC0gTkENCiAgfQ0KICBzdXBwcmVzc1dhcm5pbmdzKHsgIyBhdm9pZCBzb21lIGFubm95aW5nIG91dHB1dCB0byB0aGUgY29uc29sZSBjYXVzZWQgYnkgdGhlIHRlc3RzDQogICAgb3V0IDwtIGRhdGEuZnJhbWUoDQogICAgICAgICAgICBvcmRlcl9ieSA9IG9yZGVyX2J5LCAjIGdpdmUgYW4gZXhhbXBsZSB2YWx1ZSBvZiB0aGUgc29ydCBieSBkYXRhDQogICAgICAgICAgICBicmV1c2NoX3BhZ2FuID0gYnB0ZXN0KG1vZGVsKSRwLnZhbHVlLCAjIHAtdmFsdWUgZnJvbSBicHRlc3QNCiAgICAgICAgICAgIGdvbGRmZWxkX3F1YW5kdCA9IGdxdGVzdChtb2RlbCkkcC52YWx1ZSwNCiAgICAgICAgICAgIGhhcnZleV9jb2xsaWVyID0gaGFydnRlc3QobW9kZWwsIG9yZGVyLmJ5ID0gb3JkZXJfdmFyKSRwLnZhbHVlLA0KICAgICAgICAgICAgcmVzZXQgPSByZXNldChtb2RlbCwgcG93ZXIgPSAyOjQpJHAudmFsdWUsDQogICAgICAgICAgICByYWluYm93ID0gcmFpbnRlc3QobW9kZWwsIG9yZGVyLmJ5ID0gb3JkZXJfdmFyKSRwLnZhbHVlLA0KICAgICAgICAgICAgYm94X3Rlc3QgPSBCb3gudGVzdChvcmRlcmVkX3Jlc2lkcykkcC52YWx1ZSwNCiAgICAgICAgICAgIGxqdW5nX3Rlc3QgPSBCb3gudGVzdChvcmRlcmVkX3Jlc2lkcywgdHlwZSA9ICJManVuZyIpJHAudmFsdWUsDQogICAgICAgICAgICBkdXJiaW5fd2F0c29uID0gZHd0ZXN0KG1vZGVsLCBvcmRlci5ieSA9IG9yZGVyX3ZhcikkcC52YWx1ZSwNCiAgICAgICAgICAgIGJyZXVzY2hfZ29kZnJleSA9IGJndGVzdChtb2RlbCwgb3JkZXIuYnkgPSBvcmRlcl92YXIpJHAudmFsdWUNCiAgICAgICAgICApDQogICAgfSkNCiAgDQogIG91dFssIDI6MTBdIDwtIGFwcGx5KG91dFssIDI6MTBdLCAyLCByb3VuZCwgMykgIyByb3VuZCBwLXZhbHVlcyB0byB0aGUgMC4wMDENCiAgcm93Lm5hbWVzKG91dCkgPC0gTlVMTA0KICByZXR1cm4ob3V0KSAjIHJldHVybiB0aGUgZGF0YS5mcmFtZQ0KfQ0KDQojIE5vdyBjYWxsIHRoZSBmdW5jdGlvbg0KcmVncmVzc19iYXR0ZXJ5KG0xLCBvcmRlcl92YXIgPSBieV9ncm91cCRkYXRhW1sxNV1dJGRpc3RpZCkNCmBgYA0KDQpUaGlzIGV4YW1wbGUgbW9kZWwgc2hvd3Mgc2lnbnMgb2YgaGV0ZXJvc2tlZGFjdGljaXR5IGFuZCBwZXJoYXBzIGEgbmVlZCBmb3IgDQpwb2x5bm9taWFsIHRlcm1zLCBidXQgb3RoZXJ3aXNlIGNoZWNrcyBvdXQuIEhvd2V2ZXIsIGl0IGlzbid0IGFib3V0IGhvdyBhIHNpbmdsZSANCm1vZGVsIHBlcmZvcm1zIC0gd2Ugd2FudCB0byBhc3Nlc3MgY29sbGVjdGl2ZWx5IGhvdyB0aGUgZmlmdHkgbW9kZWxzIG9uIG91ciANCmZpZnR5IHN1YnNhbXBsZXMgcGVyZm9ybS4gDQoNClRvIHRlc3QgYWxsIG9mIHRoZW0sIHdlIHVzZSB0aGUgYHJvd3dpc2UoKWAgb3BlcmF0b3IgdG8gb3BlcmF0ZSANCm9uIGVhY2ggcm93IG9mIG91ciBgYnlfZ3JvdXBgIGRhdGEgZnJhbWUuIFdlIHRha2Ugb25lIG1vZGVsIGFuZCB0aGUgZGF0YSBzZXQgDQppdCB3YXMgZml0IHRvIGZyb20gdGhlIGBtb2RlbGAgYW5kIGBkYXRhYCBjb2x1bW5zIHJlc3BlY3RpdmVseSwgYW5kIHRoZW4gDQpwcm9kdWNlIGEgbmV3IGRhdGEuZnJhbWUgd2l0aCA1MCByb3dzIG9mIHJlZ3Jlc3Npb24gZGlhZ25vc3RpaWNzLiANCg0KDQpgYGB7cn0NCiMgR3JvdXAgdGhlIGRhdGEgYW5kIGFwcHBseSB0aGUgZnVuY3Rpb24gdG8gZWFjaCBtb2RlbA0KbW9kX3Rlc3Rfc3MxIDwtIGJ5X2dyb3VwICU+JSAgDQogIHJvd3dpc2UoKSAlPiUgDQogIGRvKHJlZ3Jlc3NfYmF0dGVyeShtb2RlbCA9IC4kYmFzZV9tb2RlbCwgb3JkZXJfdmFyID0gLiRkYXRhJHNzMSkpDQptb2RfdGVzdF9zczEkb3JkZXJfYnkgPC0gInNzMSINCg0KbW9kX3Rlc3Rfc3MxIDwtIGJpbmRfY29scyhieV9ncm91cCAlPiUgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgZHBseXI6OnNlbGVjdChncmFkZSwgc3ViamVjdCwgdGVzdF95ZWFyKSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgIG1vZF90ZXN0X3NzMSkNCg0KIyBHcm91cCB0aGUgZGF0YSBhbmQgYXBwcGx5IHRoZSBmdW5jdGlvbiB0byBlYWNoIG1vZGVsDQptb2RfdGVzdF9kaXN0aWQgPC0gYnlfZ3JvdXAgJT4lICANCiAgcm93d2lzZSgpICU+JSANCiAgZG8ocmVncmVzc19iYXR0ZXJ5KG1vZGVsID0gLiRiYXNlX21vZGVsLCBvcmRlcl92YXIgPSAuJGRhdGEkZGlzdGlkKSkNCm1vZF90ZXN0X2Rpc3RpZCRvcmRlcl9ieSA8LSAiZGlzdGlkIg0KDQptb2RfdGVzdF9kaXN0aWQgPC0gYmluZF9jb2xzKGJ5X2dyb3VwICU+JSBkcGx5cjo6c2VsZWN0KGdyYWRlLCBzdWJqZWN0LCB0ZXN0X3llYXIpLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kX3Rlc3RfZGlzdGlkKQ0KDQpgYGANCg0KSWYgeW91IHdhbnQsIHlvdSBjYW4gaW5zcGVjdCB0aGUgcmVzdWx0cyBmb3IgdGhlc2UgbW9kZWxzLiANCg0KTm93IHdlIHJlc2hhcGUgYW5kIHBsb3QNCg0KYGBge3J9DQpwbG90ZGYgPC0gYmluZF9yb3dzKG1vZF90ZXN0X3NzMSwgbW9kX3Rlc3RfZGlzdGlkKSAlPiUgDQogIGFzLmRhdGEuZnJhbWUNCg0KbmFtZXMocGxvdGRmKVs1OjEzXSA8LSBwYXN0ZTAoInBfdmFsdWUiLCAiLiIsIG5hbWVzKHBsb3RkZilbNToxM10pDQoNCnBsb3RkZiA8LSByZXNoYXBlKHBsb3RkZiwgZGlyZWN0aW9uID0gImxvbmciLCAgDQogICAgICAgICAgICAgICAgICB2YXJ5aW5nID0gNToxMywgDQogICAgICAgICAgICAgICAgICBzZXAgPSAiLiIsIG5ldy5yb3cubmFtZXMgPSBOVUxMKQ0KDQpjdXN0b21fYnJrcyA8LSBjKDAsIDAuMSwgMC4yLCAwLjUsIDAuOCwgMSkNCg0KZ2dwbG90KHBsb3RkZiwgYWVzKHggPSBwX3ZhbHVlKSkgKyBnZW9tX2hpc3RvZ3JhbShicmVha3MgPSBjdXN0b21fYnJrcykgKyANCiAgZmFjZXRfZ3JpZChvcmRlcl9ieX50aW1lKSArIHRoZW1lX2J3KCkNCmBgYA0KDQpUaGlzIGNoYXJ0IHNob3dzIHVzIGZvciBlYWNoIHRlc3QgaG93IG1hbnkgbW9kZWxzIG91dCBvZiBmaWZ0eSBhcmUgYXQgd2hpY2ggDQpwLXZhbHVlLiBUaGUgIkJyZXVzY2gtUGFnYW4iIGNvbHVtbiBzaG93cyB0aGF0IGFsbW9zdCBhbGwgbW9kZWxzIGFyZSBncm91cGVkIGF0IA0KYSBsb3cgcC12YWx1ZSAoZS5nLiBpbmRpY2F0aW5nIHRoZSBkaWFnbm9zdGljIGlzIGZpbmRpbmcgYSBwcm9ibGVtKS4gT24gdGhlIA0Kb3RoZXIgaGFuZCwgdGhlIHJhaW5ib3cgdGVzdCBzaG93cyB0aGF0IGFsbW9zdCBubyBtb2RlbHMgd2hlbiBvcmRlcmVkIGJ5IA0KYGRpc3RpZGAgYW5kIGFsbW9zdCBhbGwgbW9kZWxzIHdoZW4gb3JkZXJlZCBieSBgc3MxYCBoYXZlIGxvdyBwLXZhbHVlcyANCmluZGljYXRpbmcgdGhlIGRpYWdub3N0aWMgaGFzIGZvdW5kIGEgcHJvYmxlbS4gDQoNClRoZSBvdGhlciB0ZXN0cyBhcmUgbXVjaCBtb3JlIG1peGVkIC0gYSBzZXQgb2YgbW9kZWxzIGZhaWwgYW5kIGEgc2V0IG9mIG1vZGVscyANCnBhc3MgcXVpdGUgYSBiaXQuIFRoZSByZXN1bHRzIGFyZSBub3QgY29uY2x1c2l2ZSAtIGFjcm9zcyB0aGUgZmlmdHkgbW9kZWxzIA0KdGhlIGRpYWdub3N0aWNzIHNob3cgdGhhdCBzb21lIG1vZGVscyBmYWlsIHNvbWUgdGVzdHMgYW5kIHBhc3Mgb3RoZXJzLiANCg0KIyMgQk9OVVM6IE92ZXIgdGhlIFRvcCBPdXRsaWVyIFBsb3QNCg0KVG8gbWFrZSBhIGZ1biBwbG90Og0KT2YgaW50ZXJlc3QgdG8gdXMgaXMgdGhlIGBpcy5pbmZgIG9iamVjdCwgd2hpY2ggZm9yIGVhY2ggcm93IGluIG91ciBkYXRhIHByb3ZpZGVzIA0Kc2l4IHRydWUvZmFsc2UgdmFsdWVzIGluZGljYXRpbmcgd2hldGhlciBhbiBvYnNlcnZhdGlvbiBpcyBpbmZsdWVudGlhbCBvbiBvbmUgDQpvZiA1IG1lYXN1cmVzIHBsdXMgYW4gYWRkaXRpb25hbCBtZWFzdXJlIGZvciBldmVyeSBjb2VmZmljaWVudCBpbiBvdXIgbW9kZWwuIA0KDQpgYGB7cn0NCg0KIyBEZWZpbmUgb3V0bGllciBzdGF0dXMgYWNyb3NzIHRoZXNlIHZhcmlhYmxlcw0Kc2NoX3Njb3JlX20xJG91dGxpZXIgPC0gc2NoX3Njb3JlX20xJC5oYXQgKyBzY2hfc2NvcmVfbTEkLmNvb2tzZCArIHNjaF9zY29yZV9tMSQuc3RkLnJlc2lkDQoNCmdncGxvdChzY2hfc2NvcmVfbTEpICsgYWVzKHggPSBzczEsIHkgPSBzczIsIGNvbG9yID0gb3V0bGllcl4yLCANCiAgICAgICAgICAgICAgICAgICAgICAgIHNpemUgPSBvdXRsaWVyXjIpICsgDQogIGdlb21fcG9pbnQoYWxwaGEgPSBJKDAuMzMpKSArDQogIHNjYWxlX2NvbG91cl9ncmFkaWVudChsb3cgPSAiYmx1ZSIsIGhpZ2g9InJlZCIsIGJyZWFrcyA9IGMoMCwgMSwgNCwgOSkpKw0KICBzY2FsZV9yYWRpdXMocmFuZ2UgPSBjKDAsOSksIGJyZWFrcyA9IGMoMCwgMSwgNCwgOSkpICsNCiAgZ3VpZGVzKGNvbG9yID0gZ3VpZGVfbGVnZW5kKCksIHNpemUgPSBndWlkZV9sZWdlbmQoKSkgKw0KICBsYWJzKGNvbG9yID0gIlN0cmVuZ3RoIG9mIEluZmwuIiwgc2l6ZSA9ICJTdHJlbmd0aCBvZiBJbmZsLiIsIA0KICAgICAgIHRpdGxlID0gIkluZmx1ZW5jZSBvZiBPYnNlcnZhdGlvbnMgb24gUmVncmVzc2lvbiBMaW5lIikgKw0KICBnZW9tX3Ntb290aChzZT1GQUxTRSwgc2hvdy5sZWdlbmQgPSBGQUxTRSkgKw0KICBnZW9tX3Ntb290aChtZXRob2Q9ICJsbSIsIHNlPUZBTFNFLCBzaG93LmxlZ2VuZCA9IEZBTFNFLCANCiAgICAgICAgICAgICAgY29sb3IgPSBJKCJibGFjayIpLCBsaW5ldHlwZSA9IDIpICsNCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gbWVhbihzY2hfc2NvcmVfbTEkc3MyKSkgKyANCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gbWVhbihzY2hfc2NvcmVfbTEkc3MxKSkgKyB0aGVtZV9idygpDQoNCmBgYA0KDQoNCg==